notebooks/part-3-finetune-gliner.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# notebook copied from here:\n",
    "# https://github.com/urchade/GLiNER/blob/main/examples/finetune.ipynb\n",
    "\n",
    "import json\n",
    "from gliner import GLiNER\n",
    "\n",
    "import torch\n",
    "from tqdm import tqdm\n",
    "from transformers import get_cosine_schedule_with_warmup\n",
    "import os\n",
    "\n",
    "\n",
    "def read_jsonl(file_path):\n",
    "    with open(file_path, \"r\") as f:\n",
    "        data = [json.loads(line) for line in f]\n",
    "    return data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load NER Dataset for Fine-Tuning\n",
    "\n",
    "**TODO**: you need to load your own NER dataset here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SAMPLE DATASET\n",
    "# from https://github.com/urchade/GLiNER/blob/main/examples/sample_data.json\n",
    "# train_path = \"sample_data.json\" \n",
    "\n",
    "# with open(train_path, \"r\") as f:\n",
    "#     data = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "train_path = \"data/generated_dataset_1_train.jsonl\"\n",
    "data = []\n",
    "\n",
    "with open(train_path, \"r\") as f:\n",
    "    for line in f:\n",
    "        data.append(json.loads(line))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# loading datasets generated in part-2\n",
    "train_data = read_jsonl('data/generated_dataset_1_train.jsonl')\n",
    "train_data = [x for x in train_data if len(x[\"ner\"]) > 0]\n",
    "eval_data = read_jsonl('data/generated_dataset_1_eval.jsonl')\n",
    "len(train_data), len(eval_data)\n",
    "\n",
    "eval_data_2 = {\"samples\": eval_data, \"entity_types\": [\"regulator\", \"job title\"]}\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'tokenized_text': ['5',\n",
       "  'in',\n",
       "  '.',\n",
       "  'Specialist',\n",
       "  'dealers',\n",
       "  'in',\n",
       "  'Impressionist',\n",
       "  ',',\n",
       "  'Modern',\n",
       "  ',',\n",
       "  'Post',\n",
       "  '-',\n",
       "  'War',\n",
       "  'and',\n",
       "  'Contemporary',\n",
       "  'art',\n",
       "  '.',\n",
       "  'We',\n",
       "  'also',\n",
       "  'advise',\n",
       "  'collectors',\n",
       "  'through',\n",
       "  'our',\n",
       "  'sister',\n",
       "  'firm',\n",
       "  'Fine',\n",
       "  'Art',\n",
       "  'Brokers',\n",
       "  '.'],\n",
       " 'ner': [[4, 5, 'person'], [20, 21, 'person'], [25, 28, 'company']]}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Pre-Trained Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dolphin/miniconda3/envs/quant_lab/lib/python3.10/site-packages/transformers/convert_slow_tokenizer.py:550: UserWarning: The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option which is not implemented in the fast tokenizers. In practice this means that the fast version of the tokenizer can produce unknown tokens whereas the sentencepiece version would have converted these unknown tokens into a sequence of byte tokens matching the original piece of text.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "# available models: https://huggingface.co/urchade\n",
    "\n",
    "model = GLiNER.from_pretrained(\"urchade/gliner_small\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fine-Tuning Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from types import SimpleNamespace\n",
    "\n",
    "# Define the hyperparameters in a config variable\n",
    "config = SimpleNamespace(\n",
    "    # num_steps=10000, # number of training iteration\n",
    "    num_steps=20, # low number of steps for testing NOTE: MODIFY THIS\n",
    "    train_batch_size=2, \n",
    "    eval_every=2, # evaluation/saving steps\n",
    "    save_directory=\"logs\", # where to save checkpoints\n",
    "    warmup_ratio=0.1, # warmup steps\n",
    "    device='cpu',\n",
    "    lr_encoder=1e-5, # learning rate for the backbone\n",
    "    lr_others=5e-5, # learning rate for other parameters\n",
    "    freeze_token_rep=False, # freeze of not the backbone\n",
    "    \n",
    "    # Parameters for set_sampling_params\n",
    "    max_types=25, # maximum number of entity types during training\n",
    "    shuffle_types=True, # if shuffle or not entity types\n",
    "    random_drop=True, # randomly drop entity types\n",
    "    max_neg_type_ratio=1, # ratio of positive/negative types, 1 mean 50%/50%, 2 mean 33%/66%, 3 mean 25%/75% ...\n",
    "    max_len=384 # maximum sentence length\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def train(model, config, train_data, eval_data=None):\n",
    "    model = model.to(config.device)\n",
    "\n",
    "    # Set sampling parameters from config\n",
    "    model.set_sampling_params(\n",
    "        max_types=config.max_types, \n",
    "        shuffle_types=config.shuffle_types, \n",
    "        random_drop=config.random_drop, \n",
    "        max_neg_type_ratio=config.max_neg_type_ratio, \n",
    "        max_len=config.max_len\n",
    "    )\n",
    "    \n",
    "    model.train()\n",
    "\n",
    "    # Initialize data loaders\n",
    "    train_loader = model.create_dataloader(train_data, batch_size=config.train_batch_size, shuffle=True)\n",
    "\n",
    "    # Optimizer\n",
    "    optimizer = model.get_optimizer(config.lr_encoder, config.lr_others, config.freeze_token_rep)\n",
    "\n",
    "    pbar = tqdm(range(config.num_steps))\n",
    "\n",
    "    if config.warmup_ratio < 1:\n",
    "        num_warmup_steps = int(config.num_steps * config.warmup_ratio)\n",
    "    else:\n",
    "        num_warmup_steps = int(config.warmup_ratio)\n",
    "\n",
    "    scheduler = get_cosine_schedule_with_warmup(\n",
    "        optimizer,\n",
    "        num_warmup_steps=num_warmup_steps,\n",
    "        num_training_steps=config.num_steps\n",
    "    )\n",
    "\n",
    "    iter_train_loader = iter(train_loader)\n",
    "\n",
    "\n",
    "    all_losses = []\n",
    "    for step in pbar:\n",
    "        try:\n",
    "            x = next(iter_train_loader)\n",
    "        except StopIteration:\n",
    "            iter_train_loader = iter(train_loader)\n",
    "            x = next(iter_train_loader)\n",
    "\n",
    "        for k, v in x.items():\n",
    "            if isinstance(v, torch.Tensor):\n",
    "                x[k] = v.to(config.device)\n",
    "\n",
    "        loss = model(x)  # Forward pass\n",
    "            \n",
    "        # Check if loss is nan\n",
    "        if torch.isnan(loss):\n",
    "            continue\n",
    "\n",
    "        loss.backward()  # Compute gradients\n",
    "        optimizer.step()  # Update parameters\n",
    "        scheduler.step()  # Update learning rate schedule\n",
    "        optimizer.zero_grad()  # Reset gradients\n",
    "\n",
    "        description = f\"step: {step} | epoch: {step // len(train_loader)} | loss: {loss.item():.2f}\"\n",
    "        pbar.set_description(description)\n",
    "\n",
    "        if (step + 1) % config.eval_every == 0:\n",
    "\n",
    "            model.eval()\n",
    "            \n",
    "            if eval_data is not None:\n",
    "                eval_losses =[]\n",
    "               # results, f1 = model.evaluate(eval_data, flat_ner=True, threshold=0.5, batch_size=12,\n",
    "               #                      entity_types=eval_data[\"entity_types\"])\n",
    "\n",
    "\n",
    "                eval_loader = model.create_dataloader(eval_data, batch_size=config.train_batch_size, shuffle=True)\n",
    "                iter_eval_loader = iter(eval_loader)\n",
    "                for step in pbar:\n",
    "                    try:\n",
    "                        x = next(iter_eval_loader)\n",
    "                    except StopIteration:\n",
    "                        iter_eval_loader = iter(eval_loader)\n",
    "                        x = next(iter_eval_loader)\n",
    "            \n",
    "                    for k, v in x.items():\n",
    "                        if isinstance(v, torch.Tensor):\n",
    "                            x[k] = v.to(config.device)\n",
    "            \n",
    "                    loss = model(x)\n",
    "                    eval_losses.append(loss.detach().numpy())\n",
    "                print(f\"average loss  on this run {np.mean(eval_losses)}\")\n",
    "                all_losses.append(np.mean(eval_losses))\n",
    "                #print(f\"Step={step}\\n{results}\")\n",
    "                \n",
    "            if not os.path.exists(config.save_directory):\n",
    "                os.makedirs(config.save_directory)\n",
    "                \n",
    "            model.save_pretrained(f\"{config.save_directory}/finetuned_{step}\")\n",
    "\n",
    "            model.train()\n",
    "\n",
    "    return all_losses"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run Fine-Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step: 1 | epoch: 0 | loss: 23.85: : 21it [00:04,  4.40it/s]                                                                                                                                                     \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average loss  on this run 36.02106857299805\n",
      "average loss  on this run 42.119140625\n",
      "average loss  on this run 50.099830627441406\n",
      "average loss  on this run 47.70021057128906\n",
      "average loss  on this run 37.21070098876953\n",
      "average loss  on this run 35.04726791381836\n",
      "average loss  on this run 38.10123062133789\n",
      "average loss  on this run 28.959392547607422\n",
      "average loss  on this run 32.490692138671875\n",
      "average loss  on this run 34.85582733154297\n"
     ]
    }
   ],
   "source": [
    "# You can also define evaluation data manually here.\n",
    "# For now, evaluation only support fix entity types (but can be easily extended)\n",
    "# eval_data = {\n",
    "#     \"entity_types\": [\"Person\", 'Event Reservation'],\n",
    "#     \"samples\": data[:10]\n",
    "# }\n",
    "run_report = train(model, config, train_data, eval_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: >"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGdCAYAAACyzRGfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABRRElEQVR4nO3deVhU9f4H8PfsrDPIDgKiIuAGriCaZoqauXTLMs3KynbNre5N7/Jru2XdW2Zamplpi0uWesstU0vcE1EEN9wF2Rcd9plh5vz+AKbcWYY5s7xfzzPPUwPMvHGUeXO+3/M5EkEQBBARERFZiVTsAERERORcWD6IiIjIqlg+iIiIyKpYPoiIiMiqWD6IiIjIqlg+iIiIyKpYPoiIiMiqWD6IiIjIquRiB7ieyWRCTk4OPD09IZFIxI5DREREDSAIAsrKyhAcHAyp9PbHNmyufOTk5CA0NFTsGERERNQEWVlZCAkJue3n2Fz58PT0BFAbXq1Wi5yGiIiIGqK0tBShoaHm9/HbsbnyUb/UolarWT6IiIjsTEO2THDDKREREVkVywcRERFZFcsHERERWZXN7fkgIiLbJwgCampqYDQaxY5CViKTySCXyy0yBoPlg4iIGkWv1yM3NxeVlZViRyErc3NzQ1BQEJRKZbMeh+WDiIgazGQy4cKFC5DJZAgODoZSqeRASCcgCAL0ej0KCwtx4cIFdOjQ4Y6DxG6H5YOIiBpMr9fDZDIhNDQUbm5uYschK3J1dYVCocClS5eg1+vh4uLS5MfihlMiImq05vzWS/bLUq87//YQERGRVbF8EBERkVWxfBAREVnA8uXL4eXl1aDPfeONN9CtW7cWzWPLGlU+3njjDUgkkmtu0dHR5o9XV1dj8uTJ8PHxgYeHB8aMGYP8/HyLhyYiIiL71egjH507d0Zubq75tmfPHvPHZsyYgQ0bNuD7779HUlIScnJy8OCDD1o0MDmOrcfzsDk9F4IgiB2FiIisqNHlQy6XIzAw0Hzz9fUFAGi1WixduhRz587FoEGD0LNnTyxbtgz79u3DgQMHLB6c7NuZ/DI8/00KXlpxGJNXHkZZtUHsSETUBIIgoFJfI8qtsb+4mEwmzJkzB23btoWrqytiY2Pxww8/wGQyISQkBIsWLbrm848cOQKpVIpLly4BAObOnYuuXbvC3d0doaGheOmll1BeXm6RP0eTyYS33noLISEhUKlU6NatG37++Wfzx/V6PaZMmYKgoCC4uLigTZs2mDNnDoDa1+CNN95AWFgYVCoVgoODMXXqVPPX6nQ6vPrqq2jdujXc3d0RHx+PnTt3mj9+6dIljBo1Cq1atYK7uzs6d+6MzZs3W+T7upVGz/k4c+YMgoOD4eLigoSEBMyZMwdhYWFISUmBwWBAYmKi+XOjo6MRFhaG/fv3o0+fPjd9PJ1OB51OZ/7/0tLSJnwbZG9WHcwy//fm9DycyCnFwgk90SlYLWIqImqsKoMRnf5vqyjPfeKtYXBTNvxtbM6cOfj222/x2WefoUOHDti1axcee+wxbN26FePHj8fKlSvx4osvmj9/xYoV6NevH9q0aQOg9jTT+fPno23btjh//jxeeukl/O1vf8PChQub/b18/PHH+PDDD7F48WJ0794dX375JUaPHo3jx4+jQ4cOmD9/Pn766SesWbMGYWFhyMrKQlZW7c/RtWvX4qOPPsLq1avRuXNn5OXl4ejRo+bHnjJlCk6cOIHVq1cjODgY69evx7333ov09HR06NABkydPhl6vx65du+Du7o4TJ07Aw8Oj2d/T7TSqfMTHx2P58uWIiopCbm4u3nzzTfTv3x/Hjh1DXl4elErlDZttAgICkJeXd8vHnDNnDt58880mhSf7VG0wYt2RywCAGYmR+C45ExeLK/HAwr146/7OGNsrlBMTiciidDod3n33XWzfvh0JCQkAgHbt2mHPnj1YvHgx/va3v+HDDz9EZmYmwsLCYDKZsHr1avzzn/80P8b06dPN/x0eHo5///vfeOGFFyxSPj744AO89tprGDduHADg/fffx2+//YZ58+bh008/RWZmJjp06IC77roLEonEXIgAIDMzE4GBgUhMTIRCoUBYWBji4uLMH1u2bBkyMzMRHBwMAHj11Vfx888/Y9myZXj33XeRmZmJMWPGoGvXruY/l5bWqPIxfPhw83/HxMQgPj4ebdq0wZo1a+Dq6tqkALNnz8bMmTPN/19aWorQ0NAmPRbZh63H83C10oAgjQumDIrAEwltMGNNKnZmFOK1tek4eOEK/v2XLnBVysSOSkR34KqQ4cRbw0R77oY6e/YsKisrMWTIkGvu1+v16N69O7p164aOHTti5cqVmDVrFpKSklBQUICHH37Y/Lnbt2/HnDlzcOrUKZSWlqKmpgbV1dWorKxs1rTX0tJS5OTkoF+/ftfc369fP/MRjCeffBJDhgxBVFQU7r33XowcORJDhw4FADz88MOYN28e2rVrh3vvvRf33XcfRo0aBblcjvT0dBiNRkRGRl7z2DqdDj4+PgCAqVOn4sUXX8Qvv/yCxMREjBkzBjExMU3+fhqiWafaenl5ITIyEmfPnkVgYCD0ej2uXr16zefk5+cjMDDwlo+hUqmgVquvuZFjW1235DK2VyhkUglauSvx5cTe+OuwKEglwNrDl/GXT/fiXKFl1lKJqOVIJBK4KeWi3BpzhLR+b8amTZuQmppqvp04cQI//PADAGDChAlYuXIlAGDlypW49957zW/QFy9exMiRIxETE4O1a9ciJSUFn376KYDaAtPSevTogQsXLuDtt99GVVUVxo4di4ceeggAEBoaioyMDCxcuBCurq546aWXMGDAABgMBpSXl0MmkyElJeWa7/vkyZP4+OOPAQDPPPMMzp8/j8cffxzp6eno1asXFixY0KLfT7PKR3l5Oc6dO4egoCD07NkTCoUCO3bsMH88IyMDmZmZ5kNcRBeKKrD/fDEkEmBs7z+OcEmlEky+JwLfPhMPXw8VMvLLMHrBHmw4miNiWiJyFJ06dYJKpUJmZiYiIiKuudUfbX/00Udx7NgxpKSk4IcffsCECRPMX5+SkgKTyYQPP/wQffr0QWRkJHJyLPPzSa1WIzg4GHv37r3m/r1796JTp07XfN4jjzyCJUuW4LvvvsPatWtRUlICoPa6K6NGjcL8+fOxc+dO7N+/H+np6ejevTuMRiMKCgpu+L7/fGAgNDQUL7zwAtatW4dXXnkFS5Ysscj3diuNWnZ59dVXMWrUKLRp0wY5OTl4/fXXIZPJMH78eGg0GkyaNAkzZ86Et7c31Go1Xn75ZSQkJNxysyk5n9XJmQCAuyP90NrrxqW6vu19sXnqXXh51RH8fqEEL686guSLJfjHiI5QybkMQ0RN4+npiVdffRUzZsyAyWTCXXfdBa1Wi71790KtVmPixIkIDw9H3759MWnSJBiNRowePdr89RERETAYDFiwYAFGjRqFvXv34rPPPrNYvr/+9a94/fXX0b59e3Tr1g3Lli1DamoqVqxYAaD2TJugoCB0794dUqkU33//PQIDA+Hl5YXly5fDaDQiPj4ebm5u+Pbbb+Hq6oo2bdrAx8cHEyZMwBNPPIEPP/wQ3bt3R2FhIXbs2IGYmBiMGDEC06dPx/DhwxEZGYkrV67gt99+Q8eOHS32vd2U0AiPPPKIEBQUJCiVSqF169bCI488Ipw9e9b88aqqKuGll14SWrVqJbi5uQkPPPCAkJub25inELRarQBA0Gq1jfo6sn06g1Ho+fYvQpvXNgpb0m//98JQYxT+8/NJoc1rG4U2r20URi3YLWQWV1gpKRHdSlVVlXDixAmhqqpK7CiNZjKZhHnz5glRUVGCQqEQ/Pz8hGHDhglJSUnmz1m4cKEAQHjiiSdu+Pq5c+cKQUFBgqurqzBs2DDh66+/FgAIV65cEQRBEJYtWyZoNJoGZXn99deF2NhY8/8bjUbhjTfeEFq3bi0oFAohNjZW2LJli/njn3/+udCtWzfB3d1dUKvVwuDBg4XDhw8LgiAI69evF+Lj4wW1Wi24u7sLffr0EbZv327+Wr1eL/zf//2fEB4eLigUCiEoKEh44IEHhLS0NEEQBGHKlClC+/btBZVKJfj5+QmPP/64UFRUdNPct3v9G/P+LREE25rwVFpaCo1GA61Wy/0fDmZLei5eXHEYvh4q7J89CArZnVf9fjtVgBlrUnG10gC1ixxzx3ZDYqcAK6Qlopuprq7GhQsX0LZt22ZdUp3s0+1e/8a8f/PaLmQ1q5JrN5o+3CukQcUDAO6J9sfGl+9Ct1AvlFbX4JmvD2HOlpOoMZpaMioREbUglg+yiqySSuw+UwgAGNe7cadSh7Ryw5rnE/Bk33AAwOKk83h0ye/IL622dEwiIovo3LkzPDw8bnqr38fhzBo94ZSoKb4/lAVBAPq290EbH/dGf71SLsUbozujd7g3XlubhoMXSzBi/m58PK47+kX4tkBiIqKm27x5MwyGm182IiCAS8csH9TiaowmrDlUO9F0fFxYsx5rREwQOgZ54qUVh3EqrwyPLf0dMxIjMeWeCEilnIpKRLbhzxNI6UZcdqEWtzOjEHml1WjlpsDQzs1v/O38PPC/yf3wSK9QCAIwd9tpPLk8GcXlujt/MRFZhI2dq0BWYqnXneWDWlz9bI8xPUIsNqvDRSHD+w/F4IOHY+GikGLX6UKMmL8HKZdKLPL4RHRzCoUCAFBZWSlyEhJD/ete//egqbjsQi0qT1uNX08VAADGxVn+mj0P9QxBl9ZqvPTtYZwvqsAjiw9g1vBoTLqrLS9OR9QCZDIZvLy8UFBQ++/azc2N/9acgCAIqKysREFBAby8vCCTNe8XSZYPalHfH8qCSQB6h7dChL9nizxHdKAaP718F2atTcPGtFz8e9NJJF8swX8eioXGtXntnIhuVD+Wu76AkPPw8vK67fXaGorlg1qMySTgu0O1sz3G9W7eRtM78VDJsWB8d8S19cbbG09g6/F8nMzdg4UTeqBLa02LPjeRs5FIJAgKCoK/v/8tz+ggx6NQKJp9xKMeywe1mD1ni3D5ShU8XeS4r2tQiz+fRCLBEwnhiA3xwksrDiOzpBIPLtqHN0Z1xvi4UB4aJrIwmUxmsTcjci7ccEotpn6j6QPdW8NVab0fULGhXtg09S4MjvaHvsaEv69Px8w1R1Gpr7FaBiIiujWWD2oRReU6bDuRD6Dll1xuxstNiSVP9MKs4dGQSSVYfyQb93+yF2fyy6yehYiIrsXyQS1ibcplGIwCYkO90ClYnAsESqUSvHB3e6x8Jh7+niqcKSjH6E/24n9HskXJQ0REtVg+yOIEQcDquovIjW/kdVxaQnw7H2ya2h992/ugymDE9O9S8ff16ag2GMWORkTklFg+yOIOnC/BhaIKuCtlGBUbLHYcAICfpwrfTIrH1EERkEiAlb9n4qHP9iGzmIOSiIisjeWDLK5+o+nobsFwV9nOCVUyqQQzh0Zh+VNxaOWmwLHsUoxYsBtbj+eJHY2IyKmwfJBFXa3UY8ux2jdzMTaaNsTdkX7YNLU/eoR5oay6Bs9/k4J3Np2AwWgSOxoRkVNg+SCLWnc4G/oaEzoGqRETYrvDvYK9XPHd8wl45q62AIAluy9g/OcHkKutEjkZEZHjY/kgi6ndaFq75GIPQ70UMin+ObITPnusJzxVchy6dAUj5u/BrtOFYkcjInJoLB9kMYczr+J0fjlcFFLc36212HEa7N4ugdg49S50DlajpEKPicsOYu620zCaeMlwIqKWwPJBFrP6YO1RjxFdg+3ugm5tfNyx9sW+GB8XBkEA5u84g4lfHkRRuU7saEREDoflgyyitNqAjWm5AGqXXOyRi0KGOQ92xUePxMJVIcOes0UYMX83ki+WiB2NiMihsHyQRfyYmoMqgxER/h7o2aaV2HGa5YHuIfhpSj9E+Hsgv1SHcZ8fwOKkcxAELsMQEVkCywdZRP2Sy7jetr/RtCE6BHjix8n9cH+3YBhNAuZsOYVnv06BtpKXDyciai6WD2q29MtaHM8phVImxYM9QsSOYzHuKjnmPdIN7zzQBUqZFNtP5mPEgt1Iv6wVOxoRkV1j+aBmW1V3eu2wLoHwdleKnMayJBIJJsS3wbqX+iLU2xWXr1RhzKJ9+Gb/RS7DEBE1EcsHNUuFrgY/peYAsI2LyLWULq012PhyfwztFAC90YR//XgcU1enolxXI3Y0IiK7w/JBzbIpLRfluhqE+7ihTzsfseO0KI2rAosf74l/3NcRMqkEG47mYPQne5CRVyZ2NCIiu8LyQc1Sv+TySO8wSKX2v9H0TiQSCZ4d0A7fPdcHgWoXnC+swP2f7sH/jmSLHY2IyG6wfFCTZeSV4UjmVcilEjzU03E2mjZEr3BvbJp6F/p38EW1wYRXvj+Ky1cqxY5FRGQXWD6oyVbVnV6b2DEAfp4qkdNYn4+HCsufikOPMC8YTQJ2nCwQOxIRkV1g+aAmqTYYse7wZQDAODudaGoJMqkE93YJBABsP5kvchoiIvvA8kFNsuVYLkqra9DayxX9O/iJHUdUgzsGAAB+P1/Cs1+IiBqA5YOaZNXBLADA2F6hkDnBRtPbae/ngba+7tAbTdh9ulDsOERENo/lgxrtXGE5Dl4ogVQCjO3tXBtNb2VwtD8AYDv3fRAR3RHLBzXad8m1Rz0GRvkjSOMqchrbUL/08ltGAYwmTj4lIrodlg9qFH2NCWtTajeajo8LEzmN7egV3gqeLnKUVOiRmnVF7DhERDaN5YMaZduJfBRX6BGgVuGeKOfeaPpnCpkUA6O49EJE1BAsH9Qoq+smmj7cMxRyGf/6/Flix9ry8SvLBxHRbfHdgxoss7gSu88UAQAeceCLyDXVwEh/yKQSZOSXIauE006JiG6F5YMa7LtDtUc9+nfwRai3m8hpbI/GTYFebVoBAHZw4BgR0S2xfFCD1BhN+P5Q3UTT3txoeiuJdWe97DjFpRciolth+aAG+fVUAQrKdPBxV2JIpwCx49iswXX7Pg6cL0ZZtUHkNEREtonlgxpkdd1sjzE9Q6CU86/NrbTz80A7X3cYjIJ5fwwREV2L7yJ0R7naKuzMqF1GGMeNpndUf/SDF5ojIro5lg+6ozXJl2ESgPi23mjn5yF2HJtXP+10Z0Yhp50SEd0EywfdltEkYM2h2iUXTjRtmF5tWkHjqkBJhR5HMjntlIjoeiwfdFu7zxQi+2oVNK4K3NslUOw4dkEuk2Jg3fRXTjslIroRywfd1qqDtbM9HujeGi4Kmchp7Ef90gvnfRAR3Yjlg26poKwaO+p+c+eSS+PcHekHuVSCMwXlyCzmtFMioj9j+aBb+iHlMmpMArqHeSEq0FPsOHZF46pA73BvADzrhYjoeiwfdFMmk4Dv6mZ7jOdE0yapP+V2xymWDyKiP2P5oJs6cL4Yl4or4aGSY2RskNhx7FL9vo/fz5eglNNOiYjMWD7oplbVHfW4v1sw3JRykdPYp7a+7mjn544ak4DdpzntlIioHssH3aCkQo+tx/IAcKNpcyXyrBciohuwfNAN1h2+DL3RhC6t1ejSWiN2HLs2OLp238dvGQWcdkpEVIflg64hCIL5InLjuNG02XrWTTu9UmnAYU47JSICwPJB1zl06QrOFpTDVSHD/d2CxY5j9+QyKe4xTzvl0gsREcDyQdepn2g6MiYIni4KkdM4hj+mnXLUOhERwPJBf6KtMmBzei4AYBw3mlrM3VG1007PFpTjUnGF2HGIiETH8kFmP6Zmo9pgQlSAJ3qEeYkdx2GoXRSIa1s/7ZRHP4iIWD4IQO1G01UH6zaaxoVCIpGInMix8EJzRER/YPkgAEDaZS1O5pZCKZfige6txY7jcBLrRq0fvMBpp0RELB8EAFidXLvR9L4ugfByU4qcxvG08XFHhL8HakwCkjIKxY5DRCQqlg9Cha4GP6XmAOBG05ZkvtAcl16IyMmxfBA2HM1Bhd6Idr7uiK/bGEmWVz9q/beMQtQYTSKnISISD8sHmWd7PNKbG01bUvdQL3i5KaCtMiDlEqedEpHzYvlwcidySnH0shYKmQRjeoaIHceh1U47rVt6OcVTbonIeTWrfLz33nuQSCSYPn26+b6BAwdCIpFcc3vhhReam5NaSP1G0yGdAuDroRI5jePjvg8iIkDe1C9MTk7G4sWLERMTc8PHnn32Wbz11lvm/3dzc2vq01ALqtIbsf5INgBgPDeaWsWAyNppp+cKK3CxqALhvu5iRyIisromHfkoLy/HhAkTsGTJErRq1eqGj7u5uSEwMNB8U6vVzQ5Klrc5PRdl1TUI9XZFv/a+YsdxCmoXBeLb1U875dEPInJOTSofkydPxogRI5CYmHjTj69YsQK+vr7o0qULZs+ejcrKymaFpJZRv+TySK9QSKXcaGotg6N5oTkicm6NXnZZvXo1Dh8+jOTk5Jt+/NFHH0WbNm0QHByMtLQ0vPbaa8jIyMC6detu+vk6nQ46nc78/6WlpY2NRE1wtqAMyRevQCaV4OFeoWLHcSqJHQPw1sYTSL5YAm2VARpXXj2YiJxLo8pHVlYWpk2bhm3btsHFxeWmn/Pcc8+Z/7tr164ICgrC4MGDce7cObRv3/6Gz58zZw7efPPNRsam5lpddx2Xe6L8EaC++WtJLSPMxw0d/D1wpqAcSacLMTo2WOxIRERW1ahll5SUFBQUFKBHjx6Qy+WQy+VISkrC/PnzIZfLYTQab/ia+Ph4AMDZs2dv+pizZ8+GVqs137KysprwbVBj6GqMWHv4MgBgfByPeoiBF5ojImfWqCMfgwcPRnp6+jX3PfXUU4iOjsZrr70GmUx2w9ekpqYCAIKCgm76mCqVCioVT/G0pq3H83Gl0oBAtQvujvQTO45TSuzoj8+SzmFn3bRTuYwjd4jIeTSqfHh6eqJLly7X3Ofu7g4fHx906dIF586dw8qVK3HffffBx8cHaWlpmDFjBgYMGHDTU3JJHKvrJpqO7RXCNz2RdA9rBW93JUoq9Dh06Qr6tPMROxIRkdVY9J1HqVRi+/btGDp0KKKjo/HKK69gzJgx2LBhgyWfhprhUnEF9p0rhkQCjO3NJRexyKQSDIyqPerEpRcicjZNHjJWb+fOneb/Dg0NRVJSUnMfklrQ6uTaPTUDOvghpBWHv4kpsWMA1h3Oxo6TBfjHiE5ixyEishoec3ciBqMJ3x/iRlNb0b+DLxQyCc4XVeB8YbnYcYiIrIblw4nsOFmAonIdfD1U5rMtSDyeLgrzXg8OHCMiZ8Ly4UTqJ5o+1DMECm40tQmDomsvNMdR60TkTPgO5CSyr1Yh6XQhAGAcN5rajMS6I1CHLl2BttIgchoiIutg+XAS3yVnQRCAhHY+vJKqDQn1dkNkgAeMJgE7T3PphYicA8uHEzCaBHx/qPYsl3HcaGpz/ph2yvJBRM6B5cMJJJ0uQK62Gq3cFBjWOVDsOHSdxI61+z52ZhTAYDSJnIaIqOWxfDiBVXUXkXuwRwhcFDeOwCdxdQutnXZaWl2DQxeviB2HiKjFsXw4uILSavx6qvZwPmd72CaZVIJ7omqPfnDaKRE5A5YPB/d9ymUYTQJ6tWmFCH9PsePQLdQvvew4xX0fROT4WD4cmMkkmGd7jIsLEzkN3U7/SD8oZVJcKKrAOU47JSIHx/LhwPadK0ZWSRU8XeQY0TVI7Dh0Gx4qOeLbeQPg0gsROT6WDwe2qu6ox1+6tYarkhtNbV39wLHtPOWWiBwcy4eDKi7X4ZfjeQA428NeDK7b95Fy6QquVupFTkNE1HJYPhzU2sOXYTAKiAnRoHOwRuw41AAhrdwQHehZO+00o1DsOERELYblwwEJgoDVybWzPcZzo6ldqT/6wQvNEZEjY/lwQAcvlOB8YQXclDKMig0WOw41Qv2o9aTThZx2SkQOi+XDAdUf9RgdGwwPlVzkNNQYsSFe8HFXoqy6BskXSsSOQ0TUIlg+HIy20oDN6bkAONvDHsmkEtwTzYFjROTYWD4czPojl6GrMSE60BOxIdxoao/M005P5kMQBJHTEBFZHsuHA7l+o6lEIhE5ETVF/w61004vFlfiXGGF2HGIiCyO5cOBpGZdxam8MqjkUvylW2ux41ATuavk6NPeBwCnnRKRY2L5cCCrDtZONB3RNQgaN4XIaag5/lh64b4PInI8LB8OoqzagA1Hazeajo/nRlN7N6hu0+mhSyW4UsFpp0TkWFg+HMRPR3NQZTAiwt8Dvdq0EjsONVP9tFOTAOw8zaMfRORYWD4cxOqDtRtNx/UO5UZTB8ELzRGRo2L5cADHsrVIz9ZCKZPiwR4hYschC6kftb4roxD6Gk47JSLHwfLhAFYn1240Hdo5AN7uSpHTkKXEhnjB10OFMl0Nki9y2ikROQ6WDztXqa/Bj0dyAPAico5GKpVgULQfAF5ojogcC8uHnduUlosyXQ3CvN2Q0M5H7DhkYfUXmttxsoDTTonIYbB82Ln62R6P9A6FVMqNpo6mfwdfKOVSZJZU4mxBudhxiIgsguXDjp3OL8PhzKuQSyV4uBc3mjoiN6UcfeumnfKsFyJyFCwfdqz+qMfgjv7w93QROQ21lMHRf1xojojIEbB82KlqgxHrj2QDAMZxo6lDG1S37+Nw5hWUcNopETkAlg87tfV4Hq5WGtDayxUDOviJHYdaUGsvV3QMUtdOO83g0gsR2T+WDztVv+TycK8QyLjR1OHxQnNE5EhYPuzQhaIKHDhfAqkEGNsrVOw4ZAX1p9wmnea0UyKyfywfdqh+oundkX4I9nIVOQ1ZQ0xrDfw8VSjX1eDgBU47JSL7xvJhZ/Q1JqxNuQyAG02diVQqwaCo2qUXTjslInvH8mFnfki5jKJyPfw9VRhUdwomOYf6C83tOJXPaadEZNdYPuxItcGIBb+eAQC8OLA9FDK+fM7krrppp1klVTjDaadEZMf47mVHVv6eiVxtNYI0LryInBNyU8rRzzztlEsvRGS/WD7sRKW+Bgt3ngUATB3cAS4KmciJSAx/vtAcEZG9YvmwE8v3XURRuR5h3m54qCev4+Ks6vd9HM68guJynchpiIiahuXDDpRWG7A46TwAYHpiB+71cGJBGld0DlZDEIDfMgrFjkNE1CR8F7MDS3dfgLbKgAh/D9zfrbXYcUhkfyy9cN8HEdknlg8bd6VCj6V7LgAAZg6J5Ch1Mo9a33W6ELoao8hpiIgaj+XDxn226xzKdTXoFKTGvZ0DxY5DNqBLcO200wq9Eb+f57RTIrI/LB82rKCsGl/tuwgAeHVYJKQ86kGonXY6uG7A3K+neNYLEdkflg8btvC3c6g2mNA9zAv3RHGaKf2hft/H9pOcdkpE9oflw0ZlX63Cyt9rLyD36tAoSCQ86kF/uCvCFyq5FJevVOF0PqedEpF9YfmwUZ/8egZ6owkJ7XzQL8JX7DhkY1yVMvPfC047JSJ7w/Jhgy4WVWDNodor174yNFLkNGSrzBeaY/kgIjvD8mGDPt5xBkaTgIFRfugV7i12HLJRg6Nr930cybqKIk47JSI7wvJhY87kl+F/qdkAgFeGRImchmxZoMYFXVrXTTvlWS9EZEdYPmzM3G2nIQjAvZ0D0TVEI3YcsnH1Rz94oTkisicsHzbkWLYWW47lQSIBZnKvBzVAYt0pt7vPcNopEdkPlg8bMnfbaQDA/bHBiAzwFDkN2YMurdUIUNdOOz3AaadEZCdYPmxEyqUr+PVUAWRSCaYl8qgHNYxEIsGgaF5ojojsC8uHjfjwlwwAwEM9QtDW113kNGRPEs2n3BZw2ikR2QWWDxuw72wR9p0rhlImxdTEDmLHITvTL8IXLgopsq9W4VRemdhxiIjuiOVDZIIg4IO6ox7j40LR2stV5ERkb1wUMtxVN+2USy9EZA9YPkS2M6MQhzOvwkUhxeR7IsSOQ3aqft/Hdp5yS0R2gOVDRH8+6jExIRz+aheRE5G9qh+1fvTyVRSWcdopEdk2lg8R/XwsD8dzSuGulOH5u9uLHYfsWIDaBV1ba2qnnWbw6AcR2TaWD5EYTYJ5rseku9rC210pciKyd7zQHBHZC5YPkWw4moMzBeXQuCowqX87seOQA/hj2mkRqg2cdkpEtovlQwQGowkfba896vHcgHbQuCpETkSOoHOwGoFqF1TqjThwvljsOEREt8TyIYK1KZdxqbgSvh5KPNk3XOw45CAkEgkG/WngGBGRrWpW+XjvvfcgkUgwffp0833V1dWYPHkyfHx84OHhgTFjxiA/n2vQ9XQ1RszfcQYA8OLACLir5CInIkeS+Kd9H5x2SkS2qsnlIzk5GYsXL0ZMTMw198+YMQMbNmzA999/j6SkJOTk5ODBBx9sdlBHser3TORoqxGodsGE+DCx45CD6du+dtppjrYaJ3M57ZSIbFOTykd5eTkmTJiAJUuWoFWrVub7tVotli5dirlz52LQoEHo2bMnli1bhn379uHAgQMWC22vqvRGfPLbOQDAlEERcFHIRE5EjqZ22qkfAJ71QkS2q0nlY/LkyRgxYgQSExOvuT8lJQUGg+Ga+6OjoxEWFob9+/ff9LF0Oh1KS0uvuTmqr/dfRFG5DqHerhjbK1TsOOSg6pdetp/ivg8isk2N3nCwevVqHD58GMnJyTd8LC8vD0qlEl5eXtfcHxAQgLy8vJs+3pw5c/Dmm282NobdKas2YFFS7VGPaYMjoZRzry+1jEHRddNOs66ioKwa/p6cnEtEtqVR74BZWVmYNm0aVqxYARcXy/xAmz17NrRarfmWlZVlkce1NV/uuYirlQa083PHX7oFix2HHJi/2gWxIRoAwG88+kFENqhR5SMlJQUFBQXo0aMH5HI55HI5kpKSMH/+fMjlcgQEBECv1+Pq1avXfF1+fj4CAwNv+pgqlQpqtfqam6O5WqnHF7vPAwBmDomEXMajHtSyBnfkheaIyHY16l1w8ODBSE9PR2pqqvnWq1cvTJgwwfzfCoUCO3bsMH9NRkYGMjMzkZCQYPHw9mLxrvMo09UgOtAT93UJEjsOOYH6Uet7OO2UiGxQo/Z8eHp6okuXLtfc5+7uDh8fH/P9kyZNwsyZM+Ht7Q21Wo2XX34ZCQkJ6NOnj+VS25HCMh2W770IAHhlaBSkUom4gcgpdApSI0jjglxtNfafL8Y9Uf5iRyIiMrP48f+PPvoII0eOxJgxYzBgwAAEBgZi3bp1ln4au7Fo5zlUGYyIDfUyn4VA1NIkEol54ylPuSUiWyMRbGwMYmlpKTQaDbRard3v/8jVVuHu/+6EvsaEbybFoX8HP7EjkRP57VQBnlqejGCNC/bOGgSJhEfdiKjlNOb9mzsfW9CCX89CX2NCXFtv3BXhK3YccjIJ7X3gqpAhR1uNE7mOOz+HiOwPy0cLySyuxJrk2tOGXx0axd86yepcFDLc1aG29PJCc0RkS1g+Wsi8HadRYxIwINIPcW29xY5DTurPF5ojIrIVLB8t4GxBGf53JBsA8MqQSJHTkDO7p37a6WUtCkqrRU5DRFSL5aMFfLT9DEwCMLRTAGJDvcSOQ07M39PF/HfwV047JSIbwfJhYcdztNiUlguJBJg5lEc9SHyJdUc/OO2UiGwFy4eFfbTtNABgZEwwogPt+1Rhcgz1o9b3nC3ktFMisgksHxZ0JPMKtp8sgFQCzEjsIHYcIgBAxyBPBGtcUG0wYd+5IrHjEBGxfFjSh7/UHvUY0yME7fw8RE5DVEsikfBCc0RkU1g+LGT/uWLsOVsEhUyCqYN51INsS/2F5n49WQAbG2pMRE6I5cMCBEHA3G0ZAIBxvcMQ6u0mciKia/Vp5wM3pQx5pdU4nsNpp0QkLpYPC0g6XYjki1egkksxZVCE2HGIbuCikKF/3bTT7Rw4RkQiY/loJkEQzHs9Hu/TBgFqF5ETEd3c4OjafR+c90FEYmP5aKZfTuQjPVsLN6UMLw5sL3Ycolu6J9ofEgmQdlmLfE47JSIRsXw0g9EkYG7dUY+n+7WFj4dK5EREt+bnqUJsiBcAHv0gInGxfDTDxrQcZOSXwdNFjmf7txM7DtEd8UJzRGQLWD6aqMZowrztZwAAzw9oB42bQuRERHf2x7TTIk47JSLRsHw00brD2bhQVAFvdyWe7NdW7DhEDRId6InWXq6oNpiw9yynnRKROFg+mkBXY8THO2qPerx4d3t4qOQiJyJqmNppp7zQHBGJi+WjCdYkZyH7ahX8PVV4PKGN2HGIGqV+6eXXU/mcdkpEomD5aKQqvRELfj0LAHh5UARcFDKRExE1Tp923nBXypBfqsOxbE47JSLrY/lopG8PXEJBmQ6tvVzxSO8wseMQNZpKLkP/Dn4AOO2UiMTB8tEI5boaLEo6BwCYltgBSjn/+Mg+1e/72HGK5cNWrTqYiQcW7sWFogqxoxBZHN89G2HZngsoqdCjra87HuzeWuw4RE1WP+30WHYp8rScdmprFiedw+x16TiSeRVL95wXOw6RxbF8NJC20oDPd9f+EJie2AFyGf/oyH75eqjQPdQLAI9+2BJBEDB/xxnM2XLKfN/m9DzUGE0ipiKyPL6DNtCS3edRVl2DqABPjIoJFjsOUbPVn/Wyg6fc2gRBEPDBLxmYu632kg0zh0TC212Jkgo99p0rFjkdkWWxfDRAUbkOX+69AACYOTQSUqlE5EREzZdYVz72ni1ClZ7TTsUkCALe2XQSn/5Wu6fsnyM6YurgDrivayAA4KejOWLGI7I4lo8G+GznOVTqjYgJ0WBopwCx4xBZRGSAB1p7uUJXw2mnYjKZBPzfj8fxxZ7aX3Devr8znqm7VlT9Udatx/Kgq2FBJMfB8nEHedpqfHPgEgDglaFRkEh41IMcg0Qi+eNCc9z3IQqjScDsden45sAlSCTA+2O64vGEcPPHe4d7I1DtgjJdDZIyCsULSmRhLB938MlvZ6CrMaF3eCsM6OArdhwii/rzvg+TidNOranGaMKr3x/Fd4eyIJUAc8fG3jA7SCqVYGRMEAAuvZBjYfm4jaySSnyXnAWARz3IMcXXTTstKNPhWI5W7DhOw2A0YdrqVKw/kg25VIIF43vgge4hN/3cUbG1Sy87ThagUl9jzZhELYbl4zbm7zgDg1FA/w6+6NPOR+w4RBanksswILJ+2inPerEGXY0RL604jE3puVDKpFg4oQdG1B3duJmYEA3a+LihymDka0QOg+XjFs4VlmPt4csAak95I3JUfyy9cN9HS6s2GPHc1ynYdiIfKrkUnz/RE0M7B972ayQSiXnj6QYuvZCDYPm4hXnbz8AkAIkd/dE9rJXYcYhazD1RfpBIgOM5pcjVVokdx2FV6mvw9PJkJJ0uhKtChi+f7I2BUf4N+tr6pZekjEJoqwwtGZPIKlg+buJkbqn5N4yZQ6JETkPUsnw8VOhRV7A5cKxllFUbMPHLg9h3rhjuShm+ejoO/SIavoE9KtATkQEe0BtN2Ho8rwWTElkHy8dN1E8YHBEThE7BapHTELU884XmuPRicdoqAx5fehDJF6/A00WOb56JR1xb70Y/DpdeyJGwfFznaNZVbDuRD6kEmJHIvR7kHMzTTs8VI7+UF5qzlCsVekz44gBSs67Cy02BVc/2MR9laqz6pZd954pRVK6zZEwiq2P5uM6HdUc9Hugeggh/D5HTEFlHB38PRPh7QF9jwrB5u7ApLVfsSHavsEyH8UsO4Fh2KXzclVj9XB90aa1p8uOF+7ojJkQDo0nAlnS+PmTfWD7+5OCFEuw6XQi5VIJpgzuIHYfIaiQSCT57rCc6B6txtdKAySsPY9rqI9BWcnNjU+SXVmPc5/txKq8M/p4qfPd8H0QHNn8J94+lF5YPsm8sH3UEQcAHWzMAAGN7hyLMx03kRETWFeHvgfUv9cPUQRGQSSX4MTUHQ+clIek0x3o3RvbVKoxdvB/nCisQrHHBmucTEOHvaZHHrp8HcvBiCc9MIrvG8lFnz9kiHLxYAqVcipcHRYgdh0gUSrkUM4dGYe2LfdHO1x35pTpM/PIg/vm/dE7XbICskko8sng/LhVXItTbFd89n4BwX3eLPX6wlyviwms3q3JpjOwZyweuPerxWHwbBGlcRU5EJK5uoV7YNLU/nuwbDgD49kAmhn+8GymXSsQNZsPOF5bj4c/24/KVKrT1dcea5xMQ6m35I6ijYnmtF7J/LB+oHSt99LIWrgoZXhzYXuw4RDbBVSnDG6M7Y8Uz8QjWuOBScSUe/mw/3v/5FC/vfp3T+WUYu/gA8kqr0cHfA98916fFfokZ3jUIMqkEaZe1uFhU0SLPQdTSnL58mEwCPvyl9qjHU/3C4eepEjkRkW3pF+GLn2cMwJgeITAJwKKd53D/J3txMrdU7Gg24UROKcZ9fgBF5Tp0DFJj9XN94K92abHn8/VQoW/72mtNbUzj0Q+yT05fPjal5+JUXhk8VXI8N6Cd2HGIbJLaRYEPx8bis8d6wsddiVN5ZRj9yR4s3HkWRpMgdjzRpF2+ivFLDqCkQo+YEA1WPRsPH4+W/wWmfuYHl17IXjl1+agxmvDR9tq5Hs/0bwcvN6XIiYhs271dArF1xgAM6RQAg1HAf37OwNjF+53y8H/KpRJMWPI7tFUG9AjzwrfPxFvtZ8iwzoFQyCQ4nV+OjLwyqzwnkSU5dfn4X2oOzhdWoJWbAk/fFS52HCK74OuhwueP98QHD8fCUyVHyqUrGP7xbnxz4BIEwTmOghw4X4zHlx5Ema4G8W298fWkeKhdFFZ7fo2rAndH1o7E57h1skdOWz70NSbMqzvq8cLd7eFpxR8cRPZOIpHgoZ4h+HnGAPRt74MqgxH/+t8xTFyWjDytY49n332mEE8uO4hKvRH9O/hi+VNx8FDJrZ5jdLe6gWNpOU5T+shxOG35WHMoC5evVMHPU4UnEsLFjkNkl1p7ueLbSfF4fVQnqORS7DpdiKEfJeHH1GyHfEP89VQ+Jn11CNUGE+6J8sOSJ3rBVSkTJUtiR3+4KmS4VFyJtMtaUTIQNZVTlo9qgxELfj0DAJhyT4RoPzyIHIFUKsFT/dpi09T+iA3RoLS6BtNWp2LKyiMoqdCLHc9ifj6Wh+e/Sam9/k3nACx+vBdcFOL97HBTys1XI+bSC9kbpywf3x64hPxSHYI1LhgXFyp2HCKHEOHvgbUv9sXMIZGQSyXYlJ6LYfN24ddT+WJHa7afjuZg8srDMBgFjIoNxieP9oBSLv6Pz/qzXjam5cLkxGcdkf0R/1+PlVXoarBo5zkAwNTBHaCS86gHkaXIZVJMHdwB61/qhw7+Higs0+Hp5Ycwa20aynX2OZ79h5TLmL76CIwmAWN6hGDeI92gkNnGj86BUX7wdJEjr7Qahy5dETsOUYPZxr8gK1q+7yKKK/QI93HDmJ4hYschckhdQzTY8PJdeLZ/W0gkwOrkLNw7bxd+P18sdrRGWfl7Jv76w1GYBGB8XCj++1AMZFKJ2LHMVHIZhnUOBAD8dDRb5DREDedU5UNbZcDipNqjHtMTI23mtxciR+SikOEfIzph1bN9ENLKFZevVGHckgN4Z9MJVBtsfzz78r0X8Pf16RAE4Mm+4Xj3ga6Q2lDxqFe/9LI5PQ81RpPIaYgaxqnefZfuPo/S6hp08Pcw/4MlopbVp50Pfp4+AON6h0IQgCW7L2DUgj04lm27Z2gsTjqHNzacAAA8P6AdXh/VCRKJ7RUPAOjX3gfe7kqUVOix75x9HVki5+U05aO4XIeley4AAF4ZGmlTh06JHJ2HSo73xsRg6cRe8PVQ4UxBOf7y6V7M33HG5n5bn7/jDOZsOQUAmDooArOGR9ts8QBq99nc17V26YVnvZC9cJrysTEtFxV6IzoHq81rpERkXYM7BuCXGQNwX9dA1JgEzN12GmM+249zheViR4MgCPhgawbmbqsdPvjq0EjMHBpl08Wj3qiY2iO5Px/P4xWHyS44Tfl4IqENvn46Dm+M7mwXP0yIHJW3uxKfPtoDH4/rBrWLHEezruK+j3dj2d4Lop0uKggC3tl0Ep/8dhYA8I/7OmLKoA6iZGmK3uHeCFS7oKy6BkkZhWLHIbojpykfEokEAyL90DvcW+woRE5PIpHg/m6tsXXGAPTv4AtdjQlvbjiBx5b+juyrVVbNYjIJ+L8fj+OLumXZt+7vjGft7ArXUqkEI2OCAAAb0nJFTkN0Z05TPojI9gRpXPH103F4+y9d4KqQYd+5Ytz70S78kHLZKuPZjSYBf1+fjm8OXIJEArz3YFe7vdxC/Sb67SfyUam3z5kq5DxYPohIVBKJBI/3aYPN0/qjR5gXynQ1ePX7o3j+mxQUleta7HlrjCb89fujWJ2cBakE+PDhWIyLC2ux52tpMSEahHm7ocpgxPaTBWLHIbotlg8isgltfd3x/Qt98bd7o6CQSfDLiXwM+2gXth7Ps/hzGYwmTPsuFeuOZEMmlWD++O54sId9Dx2USCQYFVu39MKzXsjGsXwQkc2QSSV4aWAEfpx8F6IDPVFcocfz36TglTVHUVptsMhz6GqMeGnFYWxKy4VCJsHCCT0wMsYx5v6Mjm0NAEjKKIS2yjJ/XkQtgeWDiGxOp2A1fpzSDy/c3R5SCbD28GXc+9Eu7Dtb1KzHrTYY8fw3Kdh2Ih9KuRSfP9HLoU69jwr0RGSAB/RGU4scMSKyFJYPIrJJKrkMs4ZHY83zCWjj44YcbTUe/eJ3vPHTcVTpGz/LolJfg0lfJWNnRiFcFFIse7I37onyb4Hk4qqf+cGlF7JlLB9EZNN6hXtj89T+mBBfuxl0+b6LGLFgN1Kzrjb4Mcp1NXjyy2TsPVsMd6UMXz0Vh34Rvi2UWFz1Z73sO1fcoht2iZqD5YOIbJ67So53HuiK5U/1RoBahfOFFRizaB/m/pIBwx3Gs2urDHh86e84eLEEni5yfPNMPOLb+VgpufWF+7ojJkQDo0nAlmNceiHbxPJBRHZjYJQ/tk4fgNGxwTCaBMz/9SweWLgXp/PLbvr5Vyr0mPDFARzJvAovNwVWPtMHPcJaWTm19ZmXXlK59EK2qVHlY9GiRYiJiYFarYZarUZCQgK2bNli/vjAgQMhkUiuub3wwgsWD01EzsvLTYn547vjk0e7w8tNgWPZpRi5YA+W7DoP45/GsxeV6zB+yQEcyy6Fj7sSq57tg64hGhGTW8+IummnBy+WIFdr3YmxRA3RqPIREhKC9957DykpKTh06BAGDRqE+++/H8ePHzd/zrPPPovc3Fzz7T//+Y/FQxMRjYwJxi/TB+CeKD/oa0x4Z/NJjF9yAFkllcgvrcYji/fjVF4Z/D1V+O75PugYpBY7stUEe7kiru5SEps4bp1skERo5gxjb29v/Pe//8WkSZMwcOBAdOvWDfPmzWvy45WWlkKj0UCr1UKtdp4fFkTUNIIgYHVyFv698QQq9Ea4K2XwclMi+2oVgjQuWPlsH7T1dRc7ptV9s/8i/vXjccSGaPDjlLvEjkNOoDHv303e82E0GrF69WpUVFQgISHBfP+KFSvg6+uLLl26YPbs2aisrLzt4+h0OpSWll5zIyJqKIlEgvFxYdgybQDiwr1RoTci+2oVQr1dseb5BKcsHgAwvGsQpBLg6GUtLhZViB2H6Bryxn5Beno6EhISUF1dDQ8PD6xfvx6dOnUCADz66KNo06YNgoODkZaWhtdeew0ZGRlYt27dLR9vzpw5ePPNN5v+HRARAQjzccOq5/rgq30XkZp1FbOGRyPYy1XsWKLx9VChX4Qvdp8pwsa0HEwZ1EHsSERmjV520ev1yMzMhFarxQ8//IAvvvgCSUlJ5gLyZ7/++isGDx6Ms2fPon379jd9PJ1OB53uj3PRS0tLERoaymUXIqJmWnMoC3/7IQ1RAZ7YOmOA2HHIBtQvU94d6Wfxct6iyy5KpRIRERHo2bMn5syZg9jYWHz88cc3/dz4+HgAwNmzZ2/5eCqVynz2TP2NiIiab1jnQChkEmTklyEj7+anI5PzKKs2YMqqI5i9Lh0vrzpyxxk5LanZcz5MJtM1Ry7+LDU1FQAQFBTU3KchIqJG0rgqcHdk7Qh5jlt3bseytRi1YA82peVCLpVgeJdAyKUS0fI0as/H7NmzMXz4cISFhaGsrAwrV67Ezp07sXXrVpw7dw4rV67EfffdBx8fH6SlpWHGjBkYMGAAYmJiWio/ERHdxqjYIGw/mY8NaTl4ZWgkJBLx3nDI+gRBwIrfM/HWxhPQ15jQ2ssVCx7tLvqwvUaVj4KCAjzxxBPIzc2FRqNBTEwMtm7diiFDhiArKwvbt2/HvHnzUFFRgdDQUIwZMwb//Oc/Wyo7ERHdwZBOAXBVyHCpuBLp2VrEhHiJHYmspKzagNnr0rGxbtZLYscAfPBwDLzclCIna2T5WLp06S0/FhoaiqSkpGYHIiIiy3FTyjG4oz82puXip9Qclg8ncSxbiykrD+NicSXkUglmDY/GpLva2syRL17bhYjIwdVf6XZjWi5MpmbNlSQbJwgCvj1wCQ8u2oeLxZVo7eWKNS8k4Jn+7WymeABNmPNBRET2ZWCUHzxd5MgrrcahS1cQ19Zb7EjUAm5cZvHHBw/H2sQyy/V45IOIyMGp5DIM6xwIgGe9OKr6s1k21p3N8s8RHbHkiV42WTwAlg8iIqdQv/SyOT0XNSLOdyDLspdllutx2YWIyAn0be8Db3cliiv02HeuGAMi/cSORM1kT8ss1+ORDyIiJ6CQSXFfVy69OIrjOfa1zHI9lg8iIicxKqZ26eXn43nQ1RhFTkNNUb/M8sDC2mWWYI0Lvnve9pdZrsdlFyIiJ9E73BsBahXyS3VIyijE0LpNqGQfrl9mGRztjw/H2scyy/V45IOIyElIpRKMrDv6saHuDYzsw/XLLP+4ryO+mGg/yyzXY/kgInIio+vOetl+Ih+V+hqR09Cd1F6b5cZllmcH2Ncyy/VYPoiInEhMiAZh3m6oMhix/WSB2HHoNsqqDZi6OhX/WH8M+hoTBkf7Y/O0/ujZRtyLwlkCywcRkRORSCQYFRsEgGe92LLjOVqM/mQvNhzNcYhlluuxfBAROZnRsa0BAEkZhdBWGUROQ3/252WWC0UVDrPMcj2WDyIiJxMV6InIAA/ojSb8cjxP7DhU5/pllkHR/tg01TGWWa7H8kFE5ITqZ378xKUXm3D9Msvf74vGF0/0Qit3x1hmuR7LBxGRE6q/1su+c8UoKteJnMZ53WqZ5bkB7SGVOs4yy/VYPoiInFC4rztiQjQwmgRsOcalFzE40zLL9Vg+iIicVP3SC896sb4/L7PInGCZ5XosH0RETmpETO0pt8kXS5CrrRI5jXO42TLLmuf7OPwyy/VYPoiInFSwlyt6h7eCIACbOG69xZXrajDtpsss3mJHszqWDyIiJ1Y/bp1LLy3rRE4pRi3Yg5+cdJnleiwfRERObHjXIEglwNHLWlwqrhA7jsOpX2b5y8K9Tr3Mcj2WDyIiJ+broUK/CF8APPphaVxmuTWWDyIiJzfKvPTCfR+Wcv0yy+zhzr3Mcj2WDyIiJzescyAUMgky8suQkVcmdhy7JggCVv6eaV5mCapbZnn+budeZrkeywcRkZPTuCpwd6Q/AC69NEf9Msvf16dDX2PCPVF+2Mxllpti+SAiIoyKrZ35sSEtB4IgiJzG/txsmWXpxN5cZrkFudgBiIhIfEM6BcBVIcOl4kqkZ2sRE+IldiS7IAgCVh3MwhsbjkNfY0KQxgWfPNqdRzvugEc+iIgIbko5Bnfk0ktjcJml6Vg+iIgIwB9nvWxMy4XJxKWX2+EyS/Nw2YWIiAAAd0f6wVMlR662GocuXUFcW/4Gf72bLbMsGN8dvcL5Z9UYPPJBREQAABeFDMO6BALg0svN3GyZZdPU/iweTcDyQUREZvVLL5vTc1FjNImcxnZcLKrA6Jsss3hzmaVJWD6IiMisb3sfeLsrUVyhx75zxWLHsQkFZdV4/Mvfcb5uaNh3z3FoWHOxfBARkZlCJsV9Xbn0Uq9cV4Onlycjq6QKYd5u+HFKPy6zWADLBxERXWNUTO3Sy8/H86CrMYqcRjz6GhNe/DYFx7JL4e2uxFdPx8Hf00XsWA6B5YOIiK7RO9wbAWoVyqprsOt0kdhxRCEIAmatTcPuM0VwVcjw5ZO90dbXXexYDoPlg4iIriGVSjCy7ujHT0669PKfrRlYdyQbMqkECyf0QLdQL7EjORSWDyIiusHourNetp/IR6W+RuQ01vXVvotYtPMcAGDOg11xT7S/yIkcD8sHERHdICZEgzBvN1QZjNhxskDsOFazJT0Xb2w4DgB4ZUgkxvYKFTmRY2L5ICKiG0gkEvOVbp1l6eXghRJM+y4VggA8Gh+GKYMixI7ksFg+iIjopuoHjiVlFEJbZRA5Tcs6nV+GZ75Khr7GhCGdAvD2/V0gkXCOR0th+SAiopuKDlQjMsADeqMJvxzPEztOi8nVVmHilwdRWl2DHmFemD+uO2QcINaiWD6IiOiW6md+bEjLFTlJy9BWGfDkl8nI1VajnZ87lk7sDVelTOxYDo/lg4iIbmlk3dLL3rNFKC7XiZzGsnQ1Rjz39SFk5JfBz1OFr56KQyteq8UqWD6IiOiW2vq6o2trDYwmAZuPOc7Si8kkYOaao/j9Qgk8VHIsf6o3Qr3dxI7lNFg+iIjotupnfjjKtV4EQcDbm05gU1ouFDIJFj/eE52DNWLHciosH0REdFsjYmpPuU2+WIJcbZXIaZpvye7zWLb3IgDgg4dj0S/CV9xATojlg4iIbivYyxW9w1tBEIBNdr7x9MfUbLy7+RQA4B/3dcT93VqLnMg5sXwQEdEdOcLSy96zRXj1+6MAgKf7tcUz/duKnMh5sXwQEdEdDe8aBKkEOHpZi0vFFWLHabTjOVo8/00KDEYBI2KC8M8RHTlETEQsH0REdEe+Hirz3oiNdrb0klVSiSeXJaNcV4M+7bwxd2wspBwiJiqWDyIiapD6gWM/pdrP0suVCj0mLjuIwjIdogM9sfjxXlDJOURMbCwfRETUIMO6BEIhkyAjvwwZeWVix7mjKr0Rk75KxvnCCgRrXLD8qThoXBVixyKwfBARUQNpXBW4O9IfALAxzbaPftQYTXh51REczrwKtYscXz0dh0CNi9ixqA7LBxERNdio2NqZHz8dzYEgCCKnuTlBEPCvH49j+8l8KOVSLH2yNzoEeIodi/6E5YOIiBossWMAXBRSXCquRHq2Vuw4N7Xg17NYdTATEgkwf1w39A73FjsSXYflg4iIGsxdJUdixwAAtjnzY01yFuZuOw0AeHN0Z9zbJUjkRHQzLB9ERNQoo+oGjm1My4XJZDtLL7+eysfs9ekAgJcGtscTCeHiBqJbYvkgIqJGuTvSD54qOXK11Th06YrYcQAAqVlXMXnFERhNAh7s0Rp/HRYldiS6DZYPIiJqFBeFDMO6BAKwjaWXC0UVeHp5MqoMRgyI9MP7Y2I4vdTGsXwQEVGj1S+9bE7PRY3RJFqOwjIdJn55ECUVenRtrcGiCT2gkPGtzdbxFSIiokbr294H3u5KFFfosf98sSgZKnQ1eHp5MjJLKhHm7YYvn+wNd5VclCzUOCwfRETUaAqZFMPrll7EGLduMJrw4orDSM/Wwttdia+ejoOfp8rqOahpWD6IiKhJRtctvfx8PA+6GqPVnlcQBLy2Ng27ThfCVSHDl0/2Rltfd6s9PzUfywcRETVJ73BvBKhVKKuuwa7TRVZ73g9+ycC6w9mQSSX4dEJ3dAv1stpzk2WwfBARUZNIpRKMrLvSrbXOevlm/0V8+ts5AMCcB7piUHSAVZ6XLIvlg4iImqz+rJdtJ/JRqa9p0ef6+Vge/u+n4wCAmUMiMbZ3aIs+H7Uclg8iImqy2BANwrzdUGUwYsfJghZ7nuSLJZi6+ggEARgfF4aXB0W02HNRy2tU+Vi0aBFiYmKgVquhVquRkJCALVu2mD9eXV2NyZMnw8fHBx4eHhgzZgzy8/MtHpqIiGyDRCIxX+m2pZZezuSX4ZmvDkFfY0JixwC8fX9nDhGzc40qHyEhIXjvvfeQkpKCQ4cOYdCgQbj//vtx/HjtYbAZM2Zgw4YN+P7775GUlIScnBw8+OCDLRKciIhsQ/3Sy86MQmirDBZ97DxtNSZ+eRDaKgN6hHlhwfjukHOImN2TCILQrKsCeXt747///S8eeugh+Pn5YeXKlXjooYcAAKdOnULHjh2xf/9+9OnTp0GPV1paCo1GA61WC7Va3ZxoRERkJUM/SsLp/HL896EYPNzLMnsxSqsNGPvZfpzKK0M7P3esfaEvWrkrLfLYZHmNef9ucn00Go1YvXo1KioqkJCQgJSUFBgMBiQmJpo/Jzo6GmFhYdi/f/8tH0en06G0tPSaGxER2ZdR9We9pOVa5PF0NUY89/UhnMorg5+nCl89Fcfi4UAaXT7S09Ph4eEBlUqFF154AevXr0enTp2Ql5cHpVIJLy+vaz4/ICAAeXl5t3y8OXPmQKPRmG+hody9TERkb0bWLb3sPVuE4nJdsx7LZBLwypqjOHC+BB4qOZY/1Ruh3m6WiEk2otHlIyoqCqmpqfj999/x4osvYuLEiThx4kSTA8yePRtardZ8y8rKavJjERGRONr6uqNraw2MJgGbj936F86GeGfzSWxMy4VCJsHix3uic7DGQinJVjS6fCiVSkRERKBnz56YM2cOYmNj8fHHHyMwMBB6vR5Xr1695vPz8/MRGBh4y8dTqVTms2fqb0REZH/qx60356yXL3afx9I9FwAAHzwci34RvhbJRral2VuGTSYTdDodevbsCYVCgR07dpg/lpGRgczMTCQkJDT3aYiIyMaNiKk95Tb5YglytVWN/vofU7Px700nAQB/vy8a93drbdF8ZDsade3h2bNnY/jw4QgLC0NZWRlWrlyJnTt3YuvWrdBoNJg0aRJmzpwJb29vqNVqvPzyy0hISGjwmS5ERGS/gr1c0Tu8FZIvXsGmtFw8079dg79239kivPr9UQDAU/3C8WwjvpbsT6PKR0FBAZ544gnk5uZCo9EgJiYGW7duxZAhQwAAH330EaRSKcaMGQOdTodhw4Zh4cKFLRKciIhsz6jYYCRfvIINR3MaXD5O5JTiuW9SYDAKGBEThH+N6MQhYg6u2XM+LI1zPoiI7FdRuQ5x72yHSQCS/joQbXxuf6n7rJJKjFm0DwVlOsS39cZXT8fBRSGzUlqyJKvM+SAiIrqer4fKvEl04x1mflyp0GPisoMoKNMhKsATnz/Ri8XDSbB8EBGRRZkHjt3mrJdqgxHPfH0I5wsrEKRxwfKne0PjqrBWRBIZywcREVnUsC6BUMgkOJVXhtP5ZTd83GgS8PKqI0i5dAVqFzm+ejoOQRpXEZKSWFg+iIjIojSuCtwd6Q/gxqMfgiDg/348hm0n8qGUS/HFxN6IDPAUIyaJiOWDiIgsblRs7cyPDUdz8OfzGj797SxW/J4JiQSYP64b4tp6ixWRRMTyQUREFpfYMQAuCikuFlciPVsLAFhzKAsf/HIaAPDGqM64t0uQmBFJRCwfRERkce4qORI7BgCoPfrxW0YBZq9LBwC8OLA9JvYNFzEdiY3lg4iIWsSoumu9rD2cjZe+PQyjScCDPVrjb8OiRE5GYmvUhFMiIqKGujvSD54qOUoq9ACA/h188f6YGE4vJR75ICKiluGikOHeLrVXNe/SWo1Fj/WEQsa3HeKRDyIiakGzhkejU7Aaf+nWGh4qvuVQLf5NICKiFuPjocJT/dqKHYNsDI9/ERERkVWxfBAREZFVsXwQERGRVbF8EBERkVWxfBAREZFVsXwQERGRVbF8EBERkVWxfBAREZFVsXwQERGRVbF8EBERkVWxfBAREZFVsXwQERGRVbF8EBERkVXZ3FVtBUEAAJSWloqchIiIiBqq/n27/n38dmyufJSVlQEAQkNDRU5CREREjVVWVgaNRnPbz5EIDakoVmQymZCTkwNPT09IJBKLPnZpaSlCQ0ORlZUFtVpt0cemxuPrYVv4etgWvh62h6/J7QmCgLKyMgQHB0Mqvf2uDps78iGVShESEtKiz6FWq/kXx4bw9bAtfD1sC18P28PX5NbudMSjHjecEhERkVWxfBAREZFVOVX5UKlUeP3116FSqcSOQuDrYWv4etgWvh62h6+J5djchlMiIiJybE515IOIiIjEx/JBREREVsXyQURERFbF8kFERERW5TTl49NPP0V4eDhcXFwQHx+PgwcPih3Jac2ZMwe9e/eGp6cn/P398Ze//AUZGRlixyIA7733HiQSCaZPny52FKeWnZ2Nxx57DD4+PnB1dUXXrl1x6NAhsWM5JaPRiH/9619o27YtXF1d0b59e7z99tsNun4J3ZpTlI/vvvsOM2fOxOuvv47Dhw8jNjYWw4YNQ0FBgdjRnFJSUhImT56MAwcOYNu2bTAYDBg6dCgqKirEjubUkpOTsXjxYsTExIgdxalduXIF/fr1g0KhwJYtW3DixAl8+OGHaNWqldjRnNL777+PRYsW4ZNPPsHJkyfx/vvv4z//+Q8WLFggdjS75hSn2sbHx6N379745JNPANRePyY0NBQvv/wyZs2aJXI6KiwshL+/P5KSkjBgwACx4zil8vJy9OjRAwsXLsS///1vdOvWDfPmzRM7llOaNWsW9u7di927d4sdhQCMHDkSAQEBWLp0qfm+MWPGwNXVFd9++62Iyeybwx/50Ov1SElJQWJiovk+qVSKxMRE7N+/X8RkVE+r1QIAvL29RU7ivCZPnowRI0Zc8++ExPHTTz+hV69eePjhh+Hv74/u3btjyZIlYsdyWn379sWOHTtw+vRpAMDRo0exZ88eDB8+XORk9s3mLixnaUVFRTAajQgICLjm/oCAAJw6dUqkVFTPZDJh+vTp6NevH7p06SJ2HKe0evVqHD58GMnJyWJHIQDnz5/HokWLMHPmTPz9739HcnIypk6dCqVSiYkTJ4odz+nMmjULpaWliI6Ohkwmg9FoxDvvvIMJEyaIHc2uOXz5INs2efJkHDt2DHv27BE7ilPKysrCtGnTsG3bNri4uIgdh1BbyHv16oV3330XANC9e3ccO3YMn332GcuHCNasWYMVK1Zg5cqV6Ny5M1JTUzF9+nQEBwfz9WgGhy8fvr6+kMlkyM/Pv+b+/Px8BAYGipSKAGDKlCnYuHEjdu3ahZCQELHjOKWUlBQUFBSgR48e5vuMRiN27dqFTz75BDqdDjKZTMSEzicoKAidOnW65r6OHTti7dq1IiVybn/9618xa9YsjBs3DgDQtWtXXLp0CXPmzGH5aAaH3/OhVCrRs2dP7Nixw3yfyWTCjh07kJCQIGIy5yUIAqZMmYL169fj119/Rdu2bcWO5LQGDx6M9PR0pKammm+9evXChAkTkJqayuIhgn79+t1w6vnp06fRpk0bkRI5t8rKSkil175VymQymEwmkRI5Boc/8gEAM2fOxMSJE9GrVy/ExcVh3rx5qKiowFNPPSV2NKc0efJkrFy5Ej/++CM8PT2Rl5cHANBoNHB1dRU5nXPx9PS8Ya+Nu7s7fHx8uAdHJDNmzEDfvn3x7rvvYuzYsTh48CA+//xzfP7552JHc0qjRo3CO++8g7CwMHTu3BlHjhzB3Llz8fTTT4sdzb4JTmLBggVCWFiYoFQqhbi4OOHAgQNiR3JaAG56W7ZsmdjRSBCEu+++W5g2bZrYMZzahg0bhC5duggqlUqIjo4WPv/8c7EjOa3S0lJh2rRpQlhYmODi4iK0a9dO+Mc//iHodDqxo9k1p5jzQURERLbD4fd8EBERkW1h+SAiIiKrYvkgIiIiq2L5ICIiIqti+SAiIiKrYvkgIiIiq2L5ICIiIqti+SAiIiKrYvkgIiIiq2L5ICIiIqti+SAiIiKrYvkgIiIiq/p/JNjVG3fpLsAAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "report_df = pd.DataFrame(run_report, columns = [\"eval_losses\"])\n",
    "report_df.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save and Load Fine-tuned Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save_pretrained(\"custom-model-small\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loaded_model = GLiNER.from_pretrained(\"custom-model-small\", local_files_only=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluate Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "entity_types = sorted(set([t for item in eval_data for _, _, t in item['ner']]))\n",
    "entity_types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loaded_model.predict_entities(\"The Federal Reserve announced new guidelines for banking operations to enhance financial stability.\", labels=['regulator'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_str, f1 = loaded_model.evaluate(test_data=eval_data, flat_ner=True, threshold=0.5, batch_size=1, entity_types=entity_types)\n",
    "print(eval_str)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation with Nervaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_predictions = []\n",
    "for item in eval_data:\n",
    "    text = ' '.join(item[\"tokenized_text\"])\n",
    "    predictions = loaded_model.predict_entities(text, labels=entity_types)\n",
    "    eval_predictions.append(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_data_for_nervaluate(data):\n",
    "    \"\"\"\n",
    "    Example for required format:\n",
    "    true = [\n",
    "    [{\"label\": \"PER\", \"start\": 2, \"end\": 4}],\n",
    "    [{\"label\": \"LOC\", \"start\": 1, \"end\": 2},\n",
    "     {\"label\": \"LOC\", \"start\": 3, \"end\": 4}]\n",
    "    ]\n",
    "    pred = [\n",
    "        [{\"label\": \"PER\", \"start\": 2, \"end\": 4}],\n",
    "        [{\"label\": \"LOC\", \"start\": 1, \"end\": 2},\n",
    "        {\"label\": \"LOC\", \"start\": 3, \"end\": 4}]\n",
    "    ]\n",
    "    \"\"\"\n",
    "    formatted_data = []\n",
    "    for item in data:\n",
    "        formatted_entities = []\n",
    "        for start, end, label in item['ner']:\n",
    "            formatted_entities.append({\"label\": label, \"start\": start, \"end\": end})\n",
    "        formatted_data.append(formatted_entities)\n",
    "    return formatted_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "formatted_eval_data = format_data_for_nervaluate(eval_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nervaluate import Evaluator\n",
    "from pprint import pprint\n",
    "\n",
    "# true = [\n",
    "#     [{\"label\": \"PER\", \"start\": 2, \"end\": 4}],\n",
    "#     [{\"label\": \"LOC\", \"start\": 1, \"end\": 2},\n",
    "#      {\"label\": \"LOC\", \"start\": 3, \"end\": 4}]\n",
    "# ]\n",
    "\n",
    "# pred = [\n",
    "#     [{\"label\": \"PER\", \"start\": 2, \"end\": 4}],\n",
    "#     [{\"label\": \"LOC\", \"start\": 1, \"end\": 2},\n",
    "#      {\"label\": \"LOC\", \"start\": 3, \"end\": 4}]\n",
    "# ]\n",
    "\n",
    "# evaluator = Evaluator(true, pred, tags=['LOC', 'PER'])\n",
    "\n",
    "evaluator = Evaluator(formatted_eval_data, eval_predictions, tags=['tool'])\n",
    "\n",
    "# Returns overall metrics and metrics for each tag\n",
    "\n",
    "results, results_per_tag = evaluator.evaluate()\n",
    "\n",
    "pprint(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}