vllm4Qwen2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "d5a1907ffffb4b67a37a9c128babc392": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_fcb94c0d6a9d4ab38f7fa95d9a259fc6",
              "IPY_MODEL_91930f8d1d1b4c85a5b185d9080ed7e2",
              "IPY_MODEL_b30c4ee7ebf64b3cbd3f056fad66820f"
            ],
            "layout": "IPY_MODEL_19021d31424943d285a6153320b33382"
          }
        },
        "fcb94c0d6a9d4ab38f7fa95d9a259fc6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c71830f3f4e94d63ad3baa5376dca4b3",
            "placeholder": "​",
            "style": "IPY_MODEL_7706804b31e54cecaa3cdb4c9d24da2a",
            "value": ""
          }
        },
        "91930f8d1d1b4c85a5b185d9080ed7e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d06ecd9e490f499b977bd60dc392f3b2",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0386948f10ef4cb2b3b059217ae4c2e2",
            "value": 1
          }
        },
        "b30c4ee7ebf64b3cbd3f056fad66820f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5086969c31d8417f866e6bd0cc9ea6f5",
            "placeholder": "​",
            "style": "IPY_MODEL_04d4f1c349a649ab86f3fd7e3462a04b",
            "value": "Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:01&lt;00:00,  1.35s/it]\n"
          }
        },
        "19021d31424943d285a6153320b33382": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c71830f3f4e94d63ad3baa5376dca4b3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7706804b31e54cecaa3cdb4c9d24da2a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d06ecd9e490f499b977bd60dc392f3b2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0386948f10ef4cb2b3b059217ae4c2e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5086969c31d8417f866e6bd0cc9ea6f5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "04d4f1c349a649ab86f3fd7e3462a04b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y63ZLrHsGk_v",
        "outputId": "9e0970e1-4279-49a9-b7c4-b03b1cd945e2"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting vllm\n",
            "  Downloading vllm-0.6.0-cp38-abi3-manylinux1_x86_64.whl.metadata (2.2 kB)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from vllm) (5.9.5)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.10/dist-packages (from vllm) (0.1.99)\n",
            "Requirement already satisfied: numpy<2.0.0 in /usr/local/lib/python3.10/dist-packages (from vllm) (1.26.4)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from vllm) (2.32.3)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from vllm) (4.66.5)\n",
            "Requirement already satisfied: py-cpuinfo in /usr/local/lib/python3.10/dist-packages (from vllm) (9.0.0)\n",
            "Requirement already satisfied: transformers>=4.43.2 in /usr/local/lib/python3.10/dist-packages (from vllm) (4.44.2)\n",
            "Requirement already satisfied: tokenizers>=0.19.1 in /usr/local/lib/python3.10/dist-packages (from vllm) (0.19.1)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.10/dist-packages (from vllm) (3.20.3)\n",
            "Collecting fastapi (from vllm)\n",
            "  Downloading fastapi-0.114.0-py3-none-any.whl.metadata (27 kB)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from vllm) (3.10.5)\n",
            "Collecting openai>=1.0 (from vllm)\n",
            "  Downloading openai-1.44.0-py3-none-any.whl.metadata (22 kB)\n",
            "Collecting uvicorn[standard] (from vllm)\n",
            "  Downloading uvicorn-0.30.6-py3-none-any.whl.metadata (6.6 kB)\n",
            "Requirement already satisfied: pydantic>=2.8 in /usr/local/lib/python3.10/dist-packages (from vllm) (2.8.2)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.10/dist-packages (from vllm) (9.4.0)\n",
            "Requirement already satisfied: prometheus-client>=0.18.0 in /usr/local/lib/python3.10/dist-packages (from vllm) (0.20.0)\n",
            "Collecting prometheus-fastapi-instrumentator>=7.0.0 (from vllm)\n",
            "  Downloading prometheus_fastapi_instrumentator-7.0.0-py3-none-any.whl.metadata (13 kB)\n",
            "Collecting tiktoken>=0.6.0 (from vllm)\n",
            "  Downloading tiktoken-0.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.6 kB)\n",
            "Collecting lm-format-enforcer==0.10.6 (from vllm)\n",
            "  Downloading lm_format_enforcer-0.10.6-py3-none-any.whl.metadata (16 kB)\n",
            "Collecting outlines<0.1,>=0.0.43 (from vllm)\n",
            "  Downloading outlines-0.0.46-py3-none-any.whl.metadata (15 kB)\n",
            "Requirement already satisfied: typing-extensions>=4.10 in /usr/local/lib/python3.10/dist-packages (from vllm) (4.12.2)\n",
            "Requirement already satisfied: filelock>=3.10.4 in /usr/local/lib/python3.10/dist-packages (from vllm) (3.15.4)\n",
            "Collecting partial-json-parser (from vllm)\n",
            "  Downloading partial_json_parser-0.2.1.1.post4-py3-none-any.whl.metadata (6.2 kB)\n",
            "Requirement already satisfied: pyzmq in /usr/local/lib/python3.10/dist-packages (from vllm) (24.0.1)\n",
            "Collecting msgspec (from vllm)\n",
            "  Downloading msgspec-0.18.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.9 kB)\n",
            "Collecting gguf==0.9.1 (from vllm)\n",
            "  Downloading gguf-0.9.1-py3-none-any.whl.metadata (3.3 kB)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.10/dist-packages (from vllm) (8.4.0)\n",
            "Collecting mistral-common>=1.3.4 (from vllm)\n",
            "  Downloading mistral_common-1.3.4-py3-none-any.whl.metadata (4.3 kB)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from vllm) (6.0.2)\n",
            "Collecting ray>=2.9 (from vllm)\n",
            "  Downloading ray-2.35.0-cp310-cp310-manylinux2014_x86_64.whl.metadata (16 kB)\n",
            "Collecting nvidia-ml-py (from vllm)\n",
            "  Downloading nvidia_ml_py-12.560.30-py3-none-any.whl.metadata (8.6 kB)\n",
            "Requirement already satisfied: torch==2.4.0 in /usr/local/lib/python3.10/dist-packages (from vllm) (2.4.0+cu121)\n",
            "Requirement already satisfied: torchvision==0.19 in /usr/local/lib/python3.10/dist-packages (from vllm) (0.19.0+cu121)\n",
            "Collecting xformers==0.0.27.post2 (from vllm)\n",
            "  Downloading xformers-0.0.27.post2-cp310-cp310-manylinux2014_x86_64.whl.metadata (1.0 kB)\n",
            "Collecting vllm-flash-attn==2.6.1 (from vllm)\n",
            "  Downloading vllm_flash_attn-2.6.1-cp310-cp310-manylinux1_x86_64.whl.metadata (476 bytes)\n",
            "Collecting interegular>=0.3.2 (from lm-format-enforcer==0.10.6->vllm)\n",
            "  Downloading interegular-0.3.3-py37-none-any.whl.metadata (3.0 kB)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from lm-format-enforcer==0.10.6->vllm) (24.1)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch==2.4.0->vllm) (1.13.2)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch==2.4.0->vllm) (3.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch==2.4.0->vllm) (3.1.4)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch==2.4.0->vllm) (2024.6.1)\n",
            "Requirement already satisfied: jsonschema<5.0.0,>=4.21.1 in /usr/local/lib/python3.10/dist-packages (from mistral-common>=1.3.4->vllm) (4.23.0)\n",
            "Collecting sentencepiece (from vllm)\n",
            "  Downloading sentencepiece-0.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (7.7 kB)\n",
            "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai>=1.0->vllm) (3.7.1)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai>=1.0->vllm) (1.7.0)\n",
            "Collecting httpx<1,>=0.23.0 (from openai>=1.0->vllm)\n",
            "  Downloading httpx-0.27.2-py3-none-any.whl.metadata (7.1 kB)\n",
            "Collecting jiter<1,>=0.4.0 (from openai>=1.0->vllm)\n",
            "  Downloading jiter-0.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.6 kB)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai>=1.0->vllm) (1.3.1)\n",
            "Collecting lark (from outlines<0.1,>=0.0.43->vllm)\n",
            "  Downloading lark-1.2.2-py3-none-any.whl.metadata (1.8 kB)\n",
            "Requirement already satisfied: nest-asyncio in /usr/local/lib/python3.10/dist-packages (from outlines<0.1,>=0.0.43->vllm) (1.6.0)\n",
            "Requirement already satisfied: cloudpickle in /usr/local/lib/python3.10/dist-packages (from outlines<0.1,>=0.0.43->vllm) (2.2.1)\n",
            "Collecting diskcache (from outlines<0.1,>=0.0.43->vllm)\n",
            "  Downloading diskcache-5.6.3-py3-none-any.whl.metadata (20 kB)\n",
            "Requirement already satisfied: numba in /usr/local/lib/python3.10/dist-packages (from outlines<0.1,>=0.0.43->vllm) (0.60.0)\n",
            "Requirement already satisfied: referencing in /usr/local/lib/python3.10/dist-packages (from outlines<0.1,>=0.0.43->vllm) (0.35.1)\n",
            "Collecting datasets (from outlines<0.1,>=0.0.43->vllm)\n",
            "  Downloading datasets-2.21.0-py3-none-any.whl.metadata (21 kB)\n",
            "Collecting pycountry (from outlines<0.1,>=0.0.43->vllm)\n",
            "  Downloading pycountry-24.6.1-py3-none-any.whl.metadata (12 kB)\n",
            "Collecting pyairports (from outlines<0.1,>=0.0.43->vllm)\n",
            "  Downloading pyairports-2.1.1-py3-none-any.whl.metadata (1.7 kB)\n",
            "Collecting starlette<1.0.0,>=0.30.0 (from prometheus-fastapi-instrumentator>=7.0.0->vllm)\n",
            "  Downloading starlette-0.38.5-py3-none-any.whl.metadata (6.0 kB)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic>=2.8->vllm) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.20.1 in /usr/local/lib/python3.10/dist-packages (from pydantic>=2.8->vllm) (2.20.1)\n",
            "Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.10/dist-packages (from ray>=2.9->vllm) (8.1.7)\n",
            "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from ray>=2.9->vllm) (1.0.8)\n",
            "Requirement already satisfied: aiosignal in /usr/local/lib/python3.10/dist-packages (from ray>=2.9->vllm) (1.3.1)\n",
            "Requirement already satisfied: frozenlist in /usr/local/lib/python3.10/dist-packages (from ray>=2.9->vllm) (1.4.1)\n",
            "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken>=0.6.0->vllm) (2024.5.15)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->vllm) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->vllm) (3.8)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->vllm) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->vllm) (2024.8.30)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /usr/local/lib/python3.10/dist-packages (from tokenizers>=0.19.1->vllm) (0.24.6)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.43.2->vllm) (0.4.4)\n",
            "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->vllm) (2.4.0)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->vllm) (24.2.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->vllm) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->vllm) (1.9.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->vllm) (4.0.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata->vllm) (3.20.1)\n",
            "Collecting h11>=0.8 (from uvicorn[standard]->vllm)\n",
            "  Downloading h11-0.14.0-py3-none-any.whl.metadata (8.2 kB)\n",
            "Collecting httptools>=0.5.0 (from uvicorn[standard]->vllm)\n",
            "  Downloading httptools-0.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.6 kB)\n",
            "Collecting python-dotenv>=0.13 (from uvicorn[standard]->vllm)\n",
            "  Downloading python_dotenv-1.0.1-py3-none-any.whl.metadata (23 kB)\n",
            "Collecting uvloop!=0.15.0,!=0.15.1,>=0.14.0 (from uvicorn[standard]->vllm)\n",
            "  Downloading uvloop-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.9 kB)\n",
            "Collecting watchfiles>=0.13 (from uvicorn[standard]->vllm)\n",
            "  Downloading watchfiles-0.24.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.9 kB)\n",
            "Collecting websockets>=10.4 (from uvicorn[standard]->vllm)\n",
            "  Downloading websockets-13.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.7 kB)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai>=1.0->vllm) (1.2.2)\n",
            "Collecting httpcore==1.* (from httpx<1,>=0.23.0->openai>=1.0->vllm)\n",
            "  Downloading httpcore-1.0.5-py3-none-any.whl.metadata (20 kB)\n",
            "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.10/dist-packages (from jsonschema<5.0.0,>=4.21.1->mistral-common>=1.3.4->vllm) (2023.12.1)\n",
            "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from jsonschema<5.0.0,>=4.21.1->mistral-common>=1.3.4->vllm) (0.20.0)\n",
            "Collecting pyarrow>=15.0.0 (from datasets->outlines<0.1,>=0.0.43->vllm)\n",
            "  Downloading pyarrow-17.0.0-cp310-cp310-manylinux_2_28_x86_64.whl.metadata (3.3 kB)\n",
            "Collecting dill<0.3.9,>=0.3.0 (from datasets->outlines<0.1,>=0.0.43->vllm)\n",
            "  Downloading dill-0.3.8-py3-none-any.whl.metadata (10 kB)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets->outlines<0.1,>=0.0.43->vllm) (2.1.4)\n",
            "Collecting xxhash (from datasets->outlines<0.1,>=0.0.43->vllm)\n",
            "  Downloading xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\n",
            "Collecting multiprocess (from datasets->outlines<0.1,>=0.0.43->vllm)\n",
            "  Downloading multiprocess-0.70.16-py310-none-any.whl.metadata (7.2 kB)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch==2.4.0->vllm) (2.1.5)\n",
            "Requirement already satisfied: llvmlite<0.44,>=0.43.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba->outlines<0.1,>=0.0.43->vllm) (0.43.0)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch==2.4.0->vllm) (1.3.0)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->outlines<0.1,>=0.0.43->vllm) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->outlines<0.1,>=0.0.43->vllm) (2024.1)\n",
            "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->outlines<0.1,>=0.0.43->vllm) (2024.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas->datasets->outlines<0.1,>=0.0.43->vllm) (1.16.0)\n",
            "Downloading vllm-0.6.0-cp38-abi3-manylinux1_x86_64.whl (170.6 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m170.6/170.6 MB\u001b[0m \u001b[31m9.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading gguf-0.9.1-py3-none-any.whl (49 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.1/49.1 kB\u001b[0m \u001b[31m3.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading lm_format_enforcer-0.10.6-py3-none-any.whl (43 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.7/43.7 kB\u001b[0m \u001b[31m3.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading vllm_flash_attn-2.6.1-cp310-cp310-manylinux1_x86_64.whl (75.9 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.9/75.9 MB\u001b[0m \u001b[31m9.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading xformers-0.0.27.post2-cp310-cp310-manylinux2014_x86_64.whl (20.8 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m20.8/20.8 MB\u001b[0m \u001b[31m48.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading mistral_common-1.3.4-py3-none-any.whl (3.3 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.3/3.3 MB\u001b[0m \u001b[31m53.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading sentencepiece-0.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m40.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading openai-1.44.0-py3-none-any.whl (367 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m367.8/367.8 kB\u001b[0m \u001b[31m19.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading outlines-0.0.46-py3-none-any.whl (101 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m101.9/101.9 kB\u001b[0m \u001b[31m8.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading prometheus_fastapi_instrumentator-7.0.0-py3-none-any.whl (19 kB)\n",
            "Downloading ray-2.35.0-cp310-cp310-manylinux2014_x86_64.whl (65.0 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m65.0/65.0 MB\u001b[0m \u001b[31m9.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading tiktoken-0.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m39.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading fastapi-0.114.0-py3-none-any.whl (94 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m94.0/94.0 kB\u001b[0m \u001b[31m8.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading msgspec-0.18.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (210 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m210.3/210.3 kB\u001b[0m \u001b[31m16.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_ml_py-12.560.30-py3-none-any.whl (40 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.5/40.5 kB\u001b[0m \u001b[31m3.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading partial_json_parser-0.2.1.1.post4-py3-none-any.whl (9.9 kB)\n",
            "Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m2.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading httptools-0.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (341 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m341.4/341.4 kB\u001b[0m \u001b[31m27.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading httpx-0.27.2-py3-none-any.whl (76 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.4/76.4 kB\u001b[0m \u001b[31m6.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading httpcore-1.0.5-py3-none-any.whl (77 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading interegular-0.3.3-py37-none-any.whl (23 kB)\n",
            "Downloading jiter-0.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (318 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m318.9/318.9 kB\u001b[0m \u001b[31m24.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading python_dotenv-1.0.1-py3-none-any.whl (19 kB)\n",
            "Downloading starlette-0.38.5-py3-none-any.whl (71 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.4/71.4 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading uvloop-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.4 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m93.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading watchfiles-0.24.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (425 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m425.7/425.7 kB\u001b[0m \u001b[31m32.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading websockets-13.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (157 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m157.3/157.3 kB\u001b[0m \u001b[31m14.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading datasets-2.21.0-py3-none-any.whl (527 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m527.3/527.3 kB\u001b[0m \u001b[31m34.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading diskcache-5.6.3-py3-none-any.whl (45 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.5/45.5 kB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading lark-1.2.2-py3-none-any.whl (111 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m111.0/111.0 kB\u001b[0m \u001b[31m9.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading pyairports-2.1.1-py3-none-any.whl (371 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m371.7/371.7 kB\u001b[0m \u001b[31m27.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading pycountry-24.6.1-py3-none-any.whl (6.3 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.3/6.3 MB\u001b[0m \u001b[31m112.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading uvicorn-0.30.6-py3-none-any.whl (62 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.8/62.8 kB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading dill-0.3.8-py3-none-any.whl (116 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m8.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading pyarrow-17.0.0-cp310-cp310-manylinux_2_28_x86_64.whl (39.9 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m39.9/39.9 MB\u001b[0m \u001b[31m17.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading multiprocess-0.70.16-py310-none-any.whl (134 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m12.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m14.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: sentencepiece, pyairports, nvidia-ml-py, xxhash, websockets, uvloop, python-dotenv, pycountry, pyarrow, partial-json-parser, msgspec, lark, jiter, interegular, httptools, h11, gguf, diskcache, dill, watchfiles, uvicorn, tiktoken, starlette, multiprocess, httpcore, xformers, vllm-flash-attn, prometheus-fastapi-instrumentator, lm-format-enforcer, httpx, fastapi, ray, openai, mistral-common, datasets, outlines, vllm\n",
            "  Attempting uninstall: sentencepiece\n",
            "    Found existing installation: sentencepiece 0.1.99\n",
            "    Uninstalling sentencepiece-0.1.99:\n",
            "      Successfully uninstalled sentencepiece-0.1.99\n",
            "  Attempting uninstall: pyarrow\n",
            "    Found existing installation: pyarrow 14.0.2\n",
            "    Uninstalling pyarrow-14.0.2:\n",
            "      Successfully uninstalled pyarrow-14.0.2\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "cudf-cu12 24.4.1 requires pyarrow<15.0.0a0,>=14.0.1, but you have pyarrow 17.0.0 which is incompatible.\n",
            "ibis-framework 8.0.0 requires pyarrow<16,>=2, but you have pyarrow 17.0.0 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed datasets-2.21.0 dill-0.3.8 diskcache-5.6.3 fastapi-0.114.0 gguf-0.9.1 h11-0.14.0 httpcore-1.0.5 httptools-0.6.1 httpx-0.27.2 interegular-0.3.3 jiter-0.5.0 lark-1.2.2 lm-format-enforcer-0.10.6 mistral-common-1.3.4 msgspec-0.18.6 multiprocess-0.70.16 nvidia-ml-py-12.560.30 openai-1.44.0 outlines-0.0.46 partial-json-parser-0.2.1.1.post4 prometheus-fastapi-instrumentator-7.0.0 pyairports-2.1.1 pyarrow-17.0.0 pycountry-24.6.1 python-dotenv-1.0.1 ray-2.35.0 sentencepiece-0.2.0 starlette-0.38.5 tiktoken-0.7.0 uvicorn-0.30.6 uvloop-0.20.0 vllm-0.6.0 vllm-flash-attn-2.6.1 watchfiles-0.24.0 websockets-13.0.1 xformers-0.0.27.post2 xxhash-3.5.0\n"
          ]
        }
      ],
      "source": [
        "!pip install vllm"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from vllm.model_executor.models import ModelRegistry\n",
        "\n",
        "# 先保存原始的 is_embedding_model 方法，以便之后需要时可以恢复\n",
        "original_is_embedding_model = ModelRegistry.is_embedding_model\n",
        "\n",
        "# 定义一个新的方法，该方法不管传入什么都返回 True\n",
        "def always_true_is_embedding_model(model_arch: str) -> bool:\n",
        "    return True\n",
        "\n",
        "# 将 ModelRegistry 类中的 is_embedding_model 方法替换为 always_true_is_embedding_model\n",
        "ModelRegistry.is_embedding_model = always_true_is_embedding_model\n",
        "\n",
        "# 现在调用 ModelRegistry.is_embedding_model 无论如何都会返回 True\n",
        "print(ModelRegistry.is_embedding_model(\"any_model_architecture\"))  # 输出 True\n",
        "\n",
        "# 如果需要恢复原始方法，可以这样做：\n",
        "# ModelRegistry.is_embedding_model = original_is_embedding_model"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gMiRAeNXGmpv",
        "outputId": "d7eb8d6f-20ef-4a05-8d30-0083ece5f2a1"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO 09-09 14:41:22 importing.py:10] Triton not installed; certain GPU-related functions will not be available.\n",
            "True\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from typing import Iterable, List, Optional, Tuple\n",
        "\n",
        "import torch\n",
        "from torch import nn\n",
        "\n",
        "from vllm.attention import AttentionMetadata\n",
        "from vllm.model_executor.layers.pooler import Pooler, PoolingType\n",
        "from vllm.model_executor.model_loader.weight_utils import default_weight_loader, maybe_remap_kv_scale_name\n",
        "from vllm.model_executor.models.qwen2 import Qwen2Model\n",
        "from vllm.model_executor.pooling_metadata import PoolingMetadata\n",
        "from vllm.sequence import PoolerOutput\n",
        "from vllm.model_executor.models.utils import is_pp_missing_parameter, make_layers\n",
        "\n",
        "# not same as mistral one\n",
        "class MyQwen2EmbeddingModel(nn.Module):\n",
        "    \"\"\"A model that uses Llama with additional embedding functionalities.\n",
        "\n",
        "   This class encapsulates the LlamaModel and provides an interface for\n",
        "   embedding operations and customized pooling functions.\n",
        "\n",
        "   Attributes:\n",
        "       model: An instance of LlamaModel used for forward operations.\n",
        "       _pooler: An instance of Pooler used for pooling operations.\n",
        "   \"\"\"\n",
        "\n",
        "    def __init__(\n",
        "        self,\n",
        "        **kwargs,\n",
        "    ) -> None:\n",
        "        super().__init__()\n",
        "        self.model = Qwen2Model(**kwargs)\n",
        "        self._pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True)\n",
        "\n",
        "    def forward(\n",
        "        self,\n",
        "        input_ids: Optional[torch.Tensor],\n",
        "        positions: torch.Tensor,\n",
        "        kv_caches: List[torch.Tensor],\n",
        "        attn_metadata: AttentionMetadata,\n",
        "        inputs_embeds: Optional[torch.Tensor] = None,\n",
        "    ) -> torch.Tensor:\n",
        "        return self.model.forward(input_ids, positions, kv_caches,\n",
        "                                  attn_metadata, inputs_embeds)\n",
        "\n",
        "    def pooler(\n",
        "        self,\n",
        "        hidden_states: torch.Tensor,\n",
        "        pooling_metadata: PoolingMetadata,\n",
        "    ) -> Optional[PoolerOutput]:\n",
        "        return self._pooler(hidden_states, pooling_metadata)\n",
        "\n",
        "    def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):\n",
        "        stacked_params_mapping = [\n",
        "            # (param_name, shard_name, shard_id)\n",
        "            (\"qkv_proj\", \"q_proj\", \"q\"),\n",
        "            (\"qkv_proj\", \"k_proj\", \"k\"),\n",
        "            (\"qkv_proj\", \"v_proj\", \"v\"),\n",
        "            (\"gate_up_proj\", \"gate_proj\", 0),\n",
        "            (\"gate_up_proj\", \"up_proj\", 1),\n",
        "        ]\n",
        "        params_dict = dict(self.named_parameters(remove_duplicate=False))\n",
        "        for name, loaded_weight in weights:\n",
        "            if \"rotary_emb.inv_freq\" in name:\n",
        "                continue\n",
        "            # 这里改一下哦，如果有config\n",
        "            if self.model.config.tie_word_embeddings and \"lm_head.weight\" in name: # model.config 这个要改一下\n",
        "                continue\n",
        "            for (param_name, weight_name, shard_id) in stacked_params_mapping:\n",
        "                if weight_name not in name:\n",
        "                    continue\n",
        "                name = name.replace(weight_name, param_name)\n",
        "                # Skip loading extra bias for GPTQ models.\n",
        "                if name.endswith(\".bias\") and name not in params_dict:\n",
        "                    continue\n",
        "                if is_pp_missing_parameter(name, self):\n",
        "                    continue\n",
        "\n",
        "                param = params_dict[name]\n",
        "                weight_loader = param.weight_loader\n",
        "                weight_loader(param, loaded_weight, shard_id)\n",
        "                break\n",
        "            else:\n",
        "                # Skip loading extra bias for GPTQ models.\n",
        "                if name.endswith(\".bias\") and name not in params_dict:\n",
        "                    continue\n",
        "                # Remapping the name of FP8 kv-scale.\n",
        "                name = maybe_remap_kv_scale_name(name, params_dict)\n",
        "                if name is None:\n",
        "                    continue\n",
        "                if is_pp_missing_parameter(name, self):\n",
        "                    continue\n",
        "\n",
        "                param = params_dict[name]\n",
        "                weight_loader = getattr(param, \"weight_loader\",\n",
        "                                        default_weight_loader)\n",
        "                weight_loader(param, loaded_weight)\n"
      ],
      "metadata": {
        "id": "7j1sL3wHGsVm"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from vllm import ModelRegistry\n",
        "\n",
        "ModelRegistry.register_model(\"MyQwen2EmbeddingModel\", MyQwen2EmbeddingModel)"
      ],
      "metadata": {
        "id": "zt35YGhGG_LT"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!git clone https://huggingface.co/Qwen/Qwen2-0.5B-Instruct"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P0Jz-scxJoWl",
        "outputId": "a3ce7f85-cd21-4730-f761-4466f066c7b1"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'Qwen2-0.5B-Instruct'...\n",
            "remote: Enumerating objects: 36, done.\u001b[K\n",
            "remote: Counting objects: 100% (33/33), done.\u001b[K\n",
            "remote: Compressing objects: 100% (33/33), done.\u001b[K\n",
            "remote: Total 36 (delta 14), reused 0 (delta 0), pack-reused 3 (from 1)\u001b[K\n",
            "Unpacking objects: 100% (36/36), 3.60 MiB | 5.64 MiB/s, done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!cat /content/Qwen2-0.5B-Instruct/config.json"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8vQdjn5hPK7w",
        "outputId": "95e81169-2dd1-48d8-98df-001cb974ae88"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{\n",
            "  \"architectures\": [\n",
            "    \"MyQwen2EmbeddingModel\"\n",
            "  ],\n",
            "  \"attention_dropout\": 0.0,\n",
            "  \"bos_token_id\": 151643,\n",
            "  \"eos_token_id\": 151645,\n",
            "  \"hidden_act\": \"silu\",\n",
            "  \"hidden_size\": 896,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 4864,\n",
            "  \"max_position_embeddings\": 32768,\n",
            "  \"max_window_layers\": 24,\n",
            "  \"model_type\": \"qwen2\",\n",
            "  \"num_attention_heads\": 14,\n",
            "  \"num_hidden_layers\": 24,\n",
            "  \"num_key_value_heads\": 2,\n",
            "  \"rms_norm_eps\": 1e-06,\n",
            "  \"rope_theta\": 1000000.0,\n",
            "  \"sliding_window\": 32768,\n",
            "  \"tie_word_embeddings\": true,\n",
            "  \"torch_dtype\": \"bfloat16\",\n",
            "  \"transformers_version\": \"4.40.1\",\n",
            "  \"use_cache\": true,\n",
            "  \"use_sliding_window\": false,\n",
            "  \"vocab_size\": 151936\n",
            "}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "! 注意我改了！"
      ],
      "metadata": {
        "id": "DtFptPlTPS2W"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from vllm import LLM\n",
        "\n",
        "# Create an LLM.\n",
        "model = LLM(model=\"/content/Qwen2-0.5B-Instruct\", enforce_eager=True, dtype=\"float16\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 158,
          "referenced_widgets": [
            "d5a1907ffffb4b67a37a9c128babc392",
            "fcb94c0d6a9d4ab38f7fa95d9a259fc6",
            "91930f8d1d1b4c85a5b185d9080ed7e2",
            "b30c4ee7ebf64b3cbd3f056fad66820f",
            "19021d31424943d285a6153320b33382",
            "c71830f3f4e94d63ad3baa5376dca4b3",
            "7706804b31e54cecaa3cdb4c9d24da2a",
            "d06ecd9e490f499b977bd60dc392f3b2",
            "0386948f10ef4cb2b3b059217ae4c2e2",
            "5086969c31d8417f866e6bd0cc9ea6f5",
            "04d4f1c349a649ab86f3fd7e3462a04b"
          ]
        },
        "id": "x7q1U4BRHDwZ",
        "outputId": "8f78b330-1ad2-4162-cdc8-f543721e6205"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "WARNING 09-09 14:48:25 config.py:1651] Casting torch.bfloat16 to torch.float16.\n",
            "WARNING 09-09 14:48:25 config.py:378] To see benefits of async output processing, enable CUDA graph. Since, enforce-eager is enabled, async output processor cannot be used\n",
            "INFO 09-09 14:48:25 llm_engine.py:213] Initializing an LLM engine (v0.6.0) with config: model='/content/Qwen2-0.5B-Instruct', speculative_config=None, tokenizer='/content/Qwen2-0.5B-Instruct', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, rope_scaling=None, rope_theta=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.float16, max_seq_len=32768, download_dir=None, load_format=LoadFormat.AUTO, tensor_parallel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=True, kv_cache_dtype=auto, quantization_param_path=None, device_config=cuda, decoding_config=DecodingConfig(guided_decoding_backend='outlines'), observability_config=ObservabilityConfig(otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=0, served_model_name=/content/Qwen2-0.5B-Instruct, use_v2_block_manager=False, num_scheduler_steps=1, enable_prefix_caching=False, use_async_output_proc=False)\n",
            "INFO 09-09 14:48:25 model_runner.py:915] Starting to load model /content/Qwen2-0.5B-Instruct...\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading safetensors checkpoint shards:   0% Completed | 0/1 [00:00<?, ?it/s]\n"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d5a1907ffffb4b67a37a9c128babc392"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO 09-09 14:48:27 model_runner.py:926] Loading model weights took 0.9221 GB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# Sample prompts.\n",
        "prompts = [\n",
        "    \"Hello, my name is\",\n",
        "    \"The president of the United States is\",\n",
        "    \"The capital of France is\",\n",
        "    \"The future of AI is\",\n",
        "]\n",
        "\n",
        "\n",
        "# Generate embedding. The output is a list of EmbeddingRequestOutputs.\n",
        "\n",
        "outputs = model.encode(prompts)\n",
        "# Print the outputs.\n",
        "for output in outputs:\n",
        "    print(output.outputs.embedding)  # list of 4096 floats\n",
        "    print(len(output.outputs.embedding))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gCSEIlKLHPuG",
        "outputId": "84dc3fda-1176-4119-dfae-d22a0f58cdb4"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Processed prompts: 100%|██████████| 4/4 [00:00<00:00, 104.10it/s, est. speed input: 0.00 toks/s, output: 0.00 toks/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[-0.00820159912109375, 0.0137939453125, -0.01505279541015625, 0.0088043212890625, 0.0006413459777832031, 0.01136016845703125, 0.0008006095886230469, 0.048004150390625, 0.0033931732177734375, 0.0008649826049804688, -0.016143798828125, 0.0005412101745605469, -0.0127410888671875, -0.01198577880859375, 0.0004444122314453125, 0.010528564453125, -0.08929443359375, -0.0125274658203125, -0.02325439453125, 0.01087188720703125, -0.00885009765625, 0.00020551681518554688, -0.00714111328125, 0.0033435821533203125, -0.0169830322265625, -0.005542755126953125, 0.00638580322265625, -0.006439208984375, -0.0032939910888671875, -0.0160064697265625, 0.0015850067138671875, -0.0237884521484375, 0.006378173828125, -0.006793975830078125, -0.0014524459838867188, 0.0084991455078125, 0.0031070709228515625, -0.0023250579833984375, -0.0307464599609375, -0.027801513671875, -0.0799560546875, 4.410743713378906e-05, 0.01206207275390625, -0.0012359619140625, -0.0252532958984375, 0.0167388916015625, -0.0178375244140625, -0.0235443115234375, -0.01216888427734375, 0.0022640228271484375, -0.0083465576171875, 0.0028285980224609375, -0.0078887939453125, 0.006580352783203125, -0.0082855224609375, 0.0005536079406738281, -0.00830841064453125, 0.0010890960693359375, 0.00832366943359375, -0.01120758056640625, 0.03021240234375, 0.006702423095703125, -0.003772735595703125, 0.004352569580078125, 0.1573486328125, 0.01140594482421875, 0.008056640625, -0.0140380859375, 0.006504058837890625, 0.01519775390625, -0.0029754638671875, 0.00726318359375, 0.00838470458984375, -0.0001493692398071289, -0.0089569091796875, 0.0009870529174804688, 0.001995086669921875, 0.0012178421020507812, 0.0146331787109375, 0.0142059326171875, 0.0030612945556640625, -0.0243682861328125, -0.01528167724609375, -0.00411224365234375, 0.00016105175018310547, -0.0127716064453125, -0.00728607177734375, -0.004974365234375, -0.006023406982421875, -0.0037441253662109375, -0.01163482666015625, -0.00542449951171875, -0.01386260986328125, 0.02191162109375, -0.0033740997314453125, -0.004241943359375, 0.00592803955078125, 0.006717681884765625, -0.006237030029296875, 0.0208740234375, -0.0039215087890625, 0.0017919540405273438, 3.0875205993652344e-05, -0.00543975830078125, 0.00809478759765625, 0.007282257080078125, -0.004108428955078125, 0.006496429443359375, -0.0189666748046875, 0.00662994384765625, -0.007053375244140625, 0.0014047622680664062, 0.00560760498046875, 0.00577545166015625, -0.0057220458984375, 0.00972747802734375, -0.00907135009765625, 0.006908416748046875, 0.01403045654296875, -0.01265716552734375, 0.00403594970703125, -0.0033016204833984375, 0.01050567626953125, -0.006427764892578125, 0.002758026123046875, 0.00395965576171875, 0.00408935546875, -0.009979248046875, -0.001857757568359375, 0.0034427642822265625, 0.003620147705078125, -0.01528167724609375, -0.01117706298828125, 0.0031147003173828125, 0.0147552490234375, -0.00464630126953125, -0.004741668701171875, -0.00545501708984375, -0.0029048919677734375, -0.01227569580078125, 0.00888824462890625, 0.00829315185546875, -0.0033321380615234375, 0.006587982177734375, -0.006061553955078125, -0.004344940185546875, -0.0133056640625, -0.0005574226379394531, -0.007366180419921875, 0.00010448694229125977, -0.025543212890625, -0.022918701171875, -0.00681304931640625, -0.01465606689453125, 0.01059722900390625, -0.00530242919921875, -0.0430908203125, -0.0186767578125, -0.004108428955078125, 0.0008153915405273438, 0.0021305084228515625, -0.008087158203125, 0.00281524658203125, -0.01081085205078125, 0.00897216796875, -0.0142364501953125, -0.00852203369140625, -0.01091766357421875, -0.0006465911865234375, -0.0283660888671875, 0.06988525390625, -0.0037670135498046875, -0.00921630859375, -0.01258087158203125, 0.00876617431640625, -0.0023746490478515625, -0.0120391845703125, 0.01458740234375, 0.007038116455078125, -0.005214691162109375, -0.0007219314575195312, -0.007152557373046875, -0.0019235610961914062, 0.0187835693359375, -0.004962921142578125, 0.006805419921875, 0.00412750244140625, 0.0007114410400390625, 0.0171966552734375, 0.042083740234375, 0.01111602783203125, 0.0097503662109375, 0.01470184326171875, 0.00921630859375, 0.00689697265625, 0.00896453857421875, 0.00579833984375, -0.00792694091796875, -0.0257568359375, 0.0014848709106445312, 0.0007405281066894531, -0.01318359375, -0.004695892333984375, 0.004123687744140625, 0.0039520263671875, -0.0009369850158691406, 0.00917816162109375, -0.007442474365234375, 0.0321044921875, -0.0018301010131835938, 0.0084075927734375, -0.003467559814453125, -0.0113067626953125, -0.00887298583984375, 0.03326416015625, 0.00789642333984375, 0.0023097991943359375, 0.0069427490234375, -0.01071929931640625, 0.007785797119140625, -0.0068511962890625, 0.10986328125, -0.0007982254028320312, -0.000545501708984375, 0.003154754638671875, 0.00017023086547851562, 0.0038242340087890625, 0.00905609130859375, 0.009521484375, 0.0125579833984375, 0.01314544677734375, -0.00893402099609375, 0.00815582275390625, 0.0144805908203125, -0.0263214111328125, -0.0023059844970703125, 0.00827789306640625, -0.0001735687255859375, 0.0012493133544921875, 0.0019521713256835938, -0.00803375244140625, -0.020050048828125, 0.0081634521484375, 0.01241302490234375, 0.03369140625, 0.006649017333984375, 0.005138397216796875, -0.03082275390625, -0.00015556812286376953, 0.006549835205078125, 0.0024929046630859375, 0.0168914794921875, -0.002269744873046875, 0.0021266937255859375, -0.0020236968994140625, 0.0037078857421875, 0.00794219970703125, 0.014404296875, -0.01538848876953125, -0.01224517822265625, 0.0020771026611328125, -0.00922393798828125, -0.0635986328125, -0.01152801513671875, -0.016387939453125, -0.002597808837890625, -0.0672607421875, 0.000637054443359375, -0.006725311279296875, -0.005817413330078125, -0.008392333984375, 0.015411376953125, -0.001537322998046875, -0.010406494140625, 0.00013303756713867188, -0.001949310302734375, 0.0153656005859375, -0.005863189697265625, 0.01354217529296875, 0.004703521728515625, 0.02447509765625, 0.007904052734375, -0.0064849853515625, 0.01546478271484375, -0.009918212890625, -0.0022335052490234375, -0.004425048828125, 0.00354766845703125, -0.0199737548828125, 0.006317138671875, -0.017059326171875, -0.00348663330078125, 0.022308349609375, 0.00350189208984375, 0.01050567626953125, -0.01497650146484375, 0.002216339111328125, 0.00154876708984375, -0.00409698486328125, 0.0027217864990234375, -0.006298065185546875, -0.00357818603515625, -0.002155303955078125, -0.0077056884765625, -0.00937652587890625, -0.00991058349609375, 0.0030002593994140625, -0.003322601318359375, 0.0016851425170898438, -0.0060882568359375, -0.0064544677734375, 0.0067138671875, 0.0016937255859375, 0.0018138885498046875, -0.005462646484375, 0.1663818359375, 0.01412200927734375, 0.00887298583984375, -0.007595062255859375, 0.00052642822265625, -0.005321502685546875, -0.004978179931640625, -0.005069732666015625, 0.0024890899658203125, 0.00611114501953125, 0.00843048095703125, -0.0131378173828125, 0.00433349609375, -0.007564544677734375, -0.013275146484375, 0.00023889541625976562, -0.005443572998046875, -0.00942230224609375, -0.009490966796875, -0.003139495849609375, 0.00875091552734375, -0.00757598876953125, -0.016204833984375, 0.002758026123046875, -0.009552001953125, -0.016082763671875, -0.00959014892578125, -0.0023708343505859375, -0.11053466796875, 0.032623291015625, 0.0027713775634765625, -0.005741119384765625, 0.00870513916015625, 0.0014848709106445312, 0.0211029052734375, -0.008880615234375, -0.0033435821533203125, 0.0037403106689453125, 0.002414703369140625, -0.00824737548828125, -0.0139312744140625, -0.0034732818603515625, 0.007495880126953125, 0.006561279296875, 0.0082855224609375, 0.0006237030029296875, -0.0034885406494140625, -0.002185821533203125, -0.0172576904296875, 0.017791748046875, -0.00620269775390625, -0.2481689453125, 0.00013184547424316406, 0.01309967041015625, 0.005298614501953125, -0.0017862319946289062, 0.002460479736328125, 0.0163116455078125, -0.007350921630859375, 0.008636474609375, 0.0031719207763671875, 0.0029811859130859375, 0.0008625984191894531, -0.0040435791015625, 0.021728515625, 0.0109710693359375, -0.0095062255859375, -0.0120391845703125, -0.0139617919921875, 0.014404296875, -0.00792694091796875, -0.0055999755859375, 0.00405120849609375, 0.005626678466796875, -0.003448486328125, -0.038330078125, 0.07373046875, -0.013946533203125, -0.0091400146484375, 0.0016641616821289062, 0.0015058517456054688, 0.003307342529296875, 0.001399993896484375, 0.021392822265625, 0.00991058349609375, 0.004459381103515625, -0.023193359375, 0.0016374588012695312, -0.00572967529296875, -0.011505126953125, -0.0004677772521972656, -0.0047454833984375, -0.0078582763671875, -0.7197265625, -0.01303863525390625, -0.00939178466796875, -0.002605438232421875, 0.03570556640625, -0.0274658203125, -0.0159149169921875, -0.018402099609375, -9.09566879272461e-05, -0.0017719268798828125, -0.01568603515625, -0.01361083984375, 0.01171875, 0.005817413330078125, 0.01108551025390625, -0.01020050048828125, -0.003910064697265625, -0.01837158203125, 0.005748748779296875, -0.02313232421875, -0.0243682861328125, -0.0074920654296875, 0.01345062255859375, 0.004459381103515625, -0.0073089599609375, -6.860494613647461e-05, -0.0012197494506835938, 0.001659393310546875, 0.0034389495849609375, -0.0075225830078125, -0.0026397705078125, -0.0265350341796875, 0.00019502639770507812, -0.015472412109375, -0.00685882568359375, -0.017059326171875, 0.009552001953125, 0.00627899169921875, 0.005313873291015625, 0.0074920654296875, 0.0014848709106445312, -0.0171661376953125, 0.0004227161407470703, -0.00675201416015625, -0.00376129150390625, 0.039642333984375, 0.00208282470703125, -0.0003998279571533203, -0.002666473388671875, 0.00174713134765625, -0.00409698486328125, -0.01189422607421875, 0.006748199462890625, 0.0033321380615234375, -0.256591796875, 0.047576904296875, -0.0014066696166992188, 0.0183258056640625, 0.0144805908203125, -0.002880096435546875, -0.01038360595703125, -0.006137847900390625, -0.29052734375, 0.0011310577392578125, -0.0007410049438476562, -0.0035858154296875, -0.01548004150390625, -0.005977630615234375, 0.0032558441162109375, -0.00292205810546875, -0.003582000732421875, -0.0037078857421875, -0.0026988983154296875, 0.00847625732421875, 0.00814056396484375, -0.012603759765625, 0.0256195068359375, 0.00501251220703125, -0.0217437744140625, 0.015472412109375, 0.004268646240234375, 0.00289154052734375, 0.0015010833740234375, 0.00020194053649902344, 0.016876220703125, -0.00013875961303710938, 0.0017719268798828125, 0.01160430908203125, 0.0300140380859375, -0.00390625, 0.004505157470703125, 0.01259613037109375, -0.0038280487060546875, -0.0033092498779296875, 0.01444244384765625, -0.0170745849609375, -0.01300811767578125, -0.01324462890625, 0.0051422119140625, 0.0026493072509765625, 0.0087432861328125, -0.01126861572265625, 0.01262664794921875, 0.004764556884765625, -0.003116607666015625, -0.009002685546875, -0.0005483627319335938, 0.0008087158203125, 0.0250091552734375, 0.00791168212890625, 0.00508880615234375, 0.007640838623046875, 0.001995086669921875, -0.00884246826171875, 0.007114410400390625, 0.006404876708984375, -0.0008087158203125, 0.016082763671875, 0.004627227783203125, -0.008331298828125, -0.0172271728515625, -0.0088348388671875, 0.003192901611328125, -0.0225067138671875, -0.0069427490234375, 0.0010433197021484375, 0.0148162841796875, -0.0003116130828857422, 0.002384185791015625, -0.0008292198181152344, 0.0117034912109375, -0.08306884765625, -0.00685882568359375, -0.00406646728515625, -0.01023101806640625, -0.011627197265625, 0.0038738250732421875, 0.00098419189453125, -0.0017995834350585938, 0.016082763671875, 0.0214385986328125, 0.018646240234375, 0.00037598609924316406, 0.0026397705078125, 0.0028591156005859375, -0.0106201171875, 0.003116607666015625, 0.0007233619689941406, -0.01413726806640625, -0.0039825439453125, 0.00399017333984375, -0.00579071044921875, 0.0033321380615234375, 0.0004520416259765625, -0.0045318603515625, 0.0159759521484375, -0.0121002197265625, 0.0201416015625, -0.0071563720703125, 0.0121307373046875, 0.0097808837890625, -0.00673675537109375, 0.048583984375, 0.004848480224609375, -0.002620697021484375, 0.01641845703125, -0.006015777587890625, 0.00966644287109375, -0.00591278076171875, 0.0107574462890625, -0.00969696044921875, -0.0005459785461425781, -0.0163421630859375, -0.0137786865234375, -0.0921630859375, 0.0010805130004882812, 0.007015228271484375, 0.0007810592651367188, 0.00757598876953125, 0.01141357421875, 0.01198577880859375, -0.0010700225830078125, 0.0187835693359375, 0.004070281982421875, 0.0033664703369140625, 0.014923095703125, -0.0023784637451171875, 0.0113067626953125, -0.0017919540405273438, 0.01152801513671875, 0.0147552490234375, 0.00839996337890625, 0.00014209747314453125, 0.07672119140625, 0.00725555419921875, -0.00743865966796875, 0.0083465576171875, -0.006195068359375, 0.005863189697265625, -0.013214111328125, 0.0015773773193359375, -0.0013904571533203125, -0.01143646240234375, -0.0035247802734375, 0.0089569091796875, 0.0153961181640625, 0.004482269287109375, -0.0062408447265625, 0.0105743408203125, 0.00789642333984375, 0.0064697265625, 0.008148193359375, -0.038970947265625, -0.01300048828125, -0.0138397216796875, 0.00852203369140625, 0.01165771484375, -0.00885772705078125, 0.0076904296875, 0.01505279541015625, 0.005748748779296875, 0.00020003318786621094, -0.0015192031860351562, 0.013885498046875, -0.0042724609375, 0.00637054443359375, -0.000400543212890625, 0.0039215087890625, -0.007476806640625, 0.009674072265625, -0.00015246868133544922, 0.0361328125, -0.00530242919921875, -0.0079193115234375, 0.01166534423828125, -0.0013208389282226562, 0.0204315185546875, -0.0006661415100097656, -0.00490570068359375, -0.006591796875, 0.0016756057739257812, 0.005069732666015625, 0.01367950439453125, -0.00899505615234375, -0.002063751220703125, -0.001068115234375, 0.040252685546875, 0.0478515625, 0.0014858245849609375, 0.006317138671875, 0.00531768798828125, -0.00803375244140625, 0.0115509033203125, 0.01885986328125, 0.00632476806640625, 0.01305389404296875, 0.0014743804931640625, -0.005207061767578125, -0.0067138671875, -0.0034542083740234375, -0.00353240966796875, 0.005779266357421875, 0.00826263427734375, 0.00762176513671875, 0.0021572113037109375, 0.004917144775390625, 0.025360107421875, 0.05230712890625, -0.03558349609375, 0.0214691162109375, -0.0013294219970703125, 0.005764007568359375, -0.0074920654296875, -0.0028591156005859375, 0.01568603515625, -0.003265380859375, -0.004253387451171875, 0.01522064208984375, -0.01074981689453125, -0.007404327392578125, 0.002593994140625, 0.0016021728515625, -0.04925537109375, 0.00315093994140625, -0.007076263427734375, 0.005405426025390625, 0.006256103515625, -0.0011959075927734375, -0.01021575927734375, -0.0135345458984375, 0.0013942718505859375, -0.00705718994140625, -0.019256591796875, 0.0167388916015625, 0.0, 0.0076904296875, 0.0025272369384765625, 0.043212890625, -0.00388336181640625, -0.03204345703125, -0.0024700164794921875, 0.01045989990234375, 0.005405426025390625, 0.057159423828125, 0.019256591796875, 0.033050537109375, -0.0108184814453125, 0.006031036376953125, -0.00537872314453125, 0.01265716552734375, -0.01172637939453125, 0.00536346435546875, 0.010223388671875, -0.020050048828125, 0.016357421875, -0.0033473968505859375, -0.01450347900390625, 0.001842498779296875, 0.0718994140625, 0.00957489013671875, 0.004150390625, -0.0234527587890625, -0.005458831787109375, 0.0024700164794921875, -0.0252838134765625, 0.005573272705078125, -0.0029048919677734375, -0.0019626617431640625, 0.00531005859375, 0.0012502670288085938, 0.015899658203125, 0.0023441314697265625, 0.01561737060546875, -0.0439453125, 0.0015916824340820312, -0.0020599365234375, -0.002834320068359375, 0.00746917724609375, -0.0013856887817382812, -0.0121002197265625, 0.00484466552734375, -0.006893157958984375, 0.0072479248046875, -0.0024700164794921875, 0.006900787353515625, 0.0112152099609375, -0.0008997917175292969, 0.01166534423828125, -0.007137298583984375, -0.033355712890625, -0.0081787109375, 0.0002865791320800781, -0.021514892578125, -0.01485443115234375, -3.516674041748047e-06, 0.005802154541015625, 1.8656253814697266e-05, -0.00202178955078125, 0.0206298828125, 0.00787353515625, 0.00530242919921875, 0.0017099380493164062, 0.00525665283203125, -0.0063323974609375, 0.01015472412109375, 0.00260162353515625, 0.009002685546875, -0.00583648681640625, 0.005229949951171875, -0.005504608154296875, -0.0185089111328125, 0.0013360977172851562, -0.0186004638671875, -0.00543212890625, -0.004730224609375, -0.0076904296875, 0.015045166015625, -0.00818634033203125, 0.001270294189453125, -0.01096343994140625, -0.0012311935424804688, -0.0023212432861328125, 0.0021839141845703125, 0.002857208251953125, -0.002532958984375, 0.006465911865234375, -0.00629425048828125, 0.005397796630859375, 0.0013027191162109375, 0.0028476715087890625, 0.0010843276977539062, -0.004505157470703125, -0.00027060508728027344, -0.01027679443359375, -0.00370025634765625, -0.0186920166015625, -0.006465911865234375, -0.0084228515625, 0.0017518997192382812, -0.00386810302734375, 0.0014286041259765625, 0.007625579833984375, -0.0003097057342529297, -0.0142822265625, 0.0192718505859375, -0.01221466064453125, -0.00846099853515625, -0.0090179443359375, 0.0009655952453613281, -0.00017523765563964844, -0.01003265380859375, -0.0226898193359375, 0.00812530517578125, -0.020751953125, 0.0026416778564453125, 0.0021114349365234375, -0.008148193359375, 0.009124755859375, -0.0032405853271484375, 0.0189208984375, 0.01016998291015625, -0.00830841064453125, -0.0084381103515625, -0.0081787109375, -0.0019311904907226562, 0.0013208389282226562, -0.007289886474609375, 0.00826263427734375, -0.001537322998046875, -0.0191192626953125, -0.0020618438720703125, 0.0123291015625, 0.0015592575073242188, -0.002979278564453125, 0.005859375, 0.00868988037109375, 0.00433349609375, -0.00482177734375, -5.716085433959961e-05, 0.004207611083984375, -0.0166168212890625, 0.0031909942626953125, 0.0152587890625, -0.004901885986328125, -0.052825927734375, -0.009552001953125, 0.00012302398681640625, 0.00952911376953125, 0.0162200927734375, -0.0032806396484375, -0.0257415771484375, -0.007297515869140625, 0.003070831298828125, 0.0054168701171875, -0.0249176025390625, -0.01383209228515625, -0.01506805419921875, 0.01453399658203125, -0.020843505859375, -0.013641357421875, 0.0030689239501953125, 0.0182647705078125, 4.2498111724853516e-05, 0.004302978515625, 0.02972412109375, -0.01788330078125, -0.01361846923828125, 0.00031948089599609375, -0.008087158203125, 0.0034542083740234375, 0.0042266845703125, -0.011627197265625, -0.00439453125, 0.007068634033203125, 0.048919677734375, -0.0016498565673828125, 0.007396697998046875, 0.01154327392578125, -0.003162384033203125, 0.0033931732177734375, -0.00337982177734375, -0.007434844970703125, -0.0012598037719726562, 0.0006060600280761719, -0.01192474365234375, -0.001720428466796875, -0.0025691986083984375, 0.0152130126953125]\n",
            "896\n",
            "[-0.0002803802490234375, 0.0154266357421875, 0.000949859619140625, -0.035980224609375, -0.016510009765625, -0.03436279296875, -0.01018524169921875, 0.0179595947265625, -0.0178985595703125, 0.015228271484375, 0.00827789306640625, -0.0015964508056640625, -0.0124664306640625, -0.002552032470703125, -0.01375579833984375, 0.0267486572265625, -0.228515625, -0.021881103515625, 0.0268707275390625, 0.0013866424560546875, 0.0005826950073242188, 0.07366943359375, -0.01300048828125, 0.015380859375, -0.0265655517578125, -0.002330780029296875, -0.004856109619140625, -0.00460052490234375, -0.0140380859375, -0.040771484375, 0.0091552734375, -0.01338958740234375, -0.004436492919921875, -0.005451202392578125, 0.0034999847412109375, -0.00583648681640625, 0.01287841796875, -0.00946044921875, -0.019012451171875, -0.0205841064453125, 0.097412109375, 0.0094757080078125, 0.00144195556640625, 0.00038504600524902344, 0.0027751922607421875, 0.01319122314453125, 0.01369476318359375, 0.00769805908203125, -0.0328369140625, -0.04229736328125, 0.001438140869140625, -0.040740966796875, -0.01200103759765625, 0.005435943603515625, 0.006195068359375, -0.006175994873046875, -0.017242431640625, 0.00473785400390625, 0.01995849609375, 0.0216217041015625, 0.00557708740234375, 0.04132080078125, -0.005001068115234375, -0.006885528564453125, 0.08062744140625, 0.0256805419921875, -0.0019550323486328125, -0.041961669921875, 0.006900787353515625, 0.01531982421875, -0.0207366943359375, -0.0035877227783203125, 0.0269012451171875, -0.0248260498046875, 0.0106964111328125, 0.00736236572265625, 0.006378173828125, -0.01323699951171875, 0.035858154296875, 0.008544921875, -0.00421905517578125, -0.029083251953125, -0.01416015625, 0.0043182373046875, -0.0068206787109375, 0.0125274658203125, 0.020050048828125, -0.01326751708984375, -0.01483917236328125, -0.0380859375, -0.02642822265625, 0.00860595703125, 0.018402099609375, 0.1419677734375, -0.01025390625, -0.0333251953125, -0.0007786750793457031, 0.02587890625, 0.00909423828125, 0.028076171875, 0.0126495361328125, -0.01287841796875, -0.006114959716796875, 0.00399017333984375, 0.0034313201904296875, 0.016357421875, -0.00850677490234375, 0.0005145072937011719, 0.007183074951171875, -0.00030303001403808594, -0.041839599609375, -0.011871337890625, 0.0211181640625, 0.0265045166015625, -0.00699615478515625, 0.009735107421875, -0.01126861572265625, 0.00501251220703125, -0.0019016265869140625, -0.007205963134765625, -0.01152801513671875, 0.01165008544921875, 0.042022705078125, -0.016998291015625, 0.0030307769775390625, -0.00962066650390625, 0.0008115768432617188, 0.0106201171875, 0.015869140625, 0.007488250732421875, 0.00498199462890625, 0.01238250732421875, 0.01114654541015625, -0.024688720703125, 0.02923583984375, -0.01959228515625, 0.012420654296875, -0.040924072265625, 0.01389312744140625, 0.0240936279296875, 0.0142364501953125, -0.035552978515625, -0.017547607421875, -0.016326904296875, -0.0031719207763671875, 0.00290679931640625, 0.034576416015625, -0.0055389404296875, 0.0303497314453125, 0.009857177734375, -0.02020263671875, -0.0108184814453125, -0.0049591064453125, -0.01458740234375, 0.0207061767578125, 0.006500244140625, 0.01239776611328125, 0.01091766357421875, 0.0292510986328125, 0.005657196044921875, 0.02117919921875, -0.0158843994140625, -0.00992584228515625, 0.025390625, 0.01837158203125, 0.019134521484375, -0.0005059242248535156, 0.0006837844848632812, -0.0020732879638671875, 0.01367950439453125, -0.062744140625, 0.0019388198852539062, 0.00881195068359375, 0.030548095703125, 0.003620147705078125, -0.0084228515625, 0.0025272369384765625, -0.01076507568359375, -0.014556884765625, -0.019989013671875, -0.00848388671875, -0.0011243820190429688, -0.00836181640625, 0.043182373046875, 0.01439666748046875, -0.0343017578125, -0.004825592041015625, -0.0016107559204101562, 0.00732421875, 0.006317138671875, -0.0367431640625, -0.0018968582153320312, -0.026458740234375, 0.011444091796875, 0.00824737548828125, -0.007602691650390625, 0.0156707763671875, 0.0029163360595703125, 0.040557861328125, -0.0296630859375, -0.0058441162109375, 0.041259765625, -0.0078277587890625, 0.00395965576171875, -0.01427459716796875, -0.01023101806640625, -0.019195556640625, 0.01123046875, 0.2293701171875, -0.01123046875, 0.02197265625, -0.0213470458984375, -0.00984954833984375, 0.013458251953125, 0.0110931396484375, 0.00630950927734375, -0.0128326416015625, 0.00797271728515625, 0.0207672119140625, 0.003215789794921875, -0.01398468017578125, 0.1029052734375, -0.004241943359375, -0.0018825531005859375, -0.006267547607421875, 0.005283355712890625, 0.005615234375, 0.01206207275390625, 0.018829345703125, 0.0119476318359375, -0.0115509033203125, 0.0113677978515625, 0.043792724609375, -0.016845703125, 0.009246826171875, -0.023590087890625, 0.00565338134765625, 0.0038890838623046875, -0.00873565673828125, 0.001995086669921875, -0.00272369384765625, -0.006999969482421875, -0.0008635520935058594, -0.00431060791015625, 0.0107421875, 0.021331787109375, -0.0196075439453125, -0.0281829833984375, -0.035919189453125, -0.0377197265625, 0.006839752197265625, -0.0174560546875, -0.00347137451171875, 0.0196075439453125, 0.005329132080078125, 0.0362548828125, 0.01447296142578125, 0.01434326171875, 0.00563812255859375, 0.0202178955078125, 0.0013952255249023438, 0.016571044921875, 0.02374267578125, -0.01007080078125, 0.01004791259765625, 0.01776123046875, -0.020660400390625, 0.00530242919921875, 0.0307769775390625, 0.0031871795654296875, -0.022186279296875, 0.06256103515625, 0.01220703125, -0.016937255859375, 0.00016260147094726562, -0.003223419189453125, 0.01227569580078125, 0.0084686279296875, 0.0099029541015625, -0.025665283203125, 0.0248870849609375, 0.020660400390625, -0.00830078125, -0.007465362548828125, -0.01477813720703125, -0.002834320068359375, 0.0137939453125, -0.046600341796875, -0.0025005340576171875, -0.00821685791015625, -0.005680084228515625, -0.0190887451171875, 0.044586181640625, 0.0188446044921875, -0.006710052490234375, -0.027587890625, -0.0008349418640136719, 0.0109405517578125, 0.00037980079650878906, -0.00135040283203125, 0.058868408203125, 0.003498077392578125, 0.0004851818084716797, 0.00499725341796875, -0.0007653236389160156, 0.01157379150390625, 0.0208282470703125, -0.0115203857421875, 0.004863739013671875, -0.004932403564453125, -0.01313018798828125, 0.004467010498046875, -0.030517578125, -0.0067138671875, 0.01275634765625, 0.31884765625, 0.0313720703125, -0.0078277587890625, 0.0209197998046875, 0.004913330078125, -0.0003573894500732422, -0.02496337890625, -0.0033092498779296875, -0.019500732421875, 0.0153045654296875, 0.006992340087890625, -0.01537322998046875, 0.02325439453125, -0.0166015625, 0.0014982223510742188, -0.0162353515625, 0.016021728515625, 0.0081939697265625, 0.005283355712890625, 0.0877685546875, -0.00749969482421875, 0.01091766357421875, 0.019775390625, -0.01268768310546875, 0.01538848876953125, 0.0087127685546875, 0.0106353759765625, 0.00787353515625, 0.08087158203125, -0.0087890625, -0.006351470947265625, -0.017486572265625, 0.00614166259765625, -0.011077880859375, 0.0095672607421875, -0.00791168212890625, 0.0452880859375, -0.032623291015625, 0.00881195068359375, 0.0166168212890625, 0.009063720703125, 0.023162841796875, 0.03448486328125, 0.00864410400390625, 0.013275146484375, 0.0157928466796875, -0.0231170654296875, -0.03009033203125, 0.02032470703125, -0.009796142578125, 0.004302978515625, -0.1666259765625, -0.03082275390625, 0.014923095703125, -0.0175933837890625, -0.004505157470703125, 0.01116180419921875, -0.0091094970703125, 0.00952911376953125, -0.0015974044799804688, -0.0012807846069335938, 0.0011930465698242188, 0.0190582275390625, 2.962350845336914e-05, 0.021484375, 0.0260009765625, -0.0188751220703125, 0.0034732818603515625, 0.006572723388671875, 0.01435089111328125, 0.006580352783203125, -0.01053619384765625, -0.02154541015625, -0.03472900390625, -0.03265380859375, -0.109130859375, 0.041290283203125, 0.0006189346313476562, -0.0019989013671875, 0.017303466796875, 0.0081024169921875, -0.0001919269561767578, -0.01471710205078125, 0.0289764404296875, 0.078369140625, 0.007411956787109375, 0.0263519287109375, -0.0031642913818359375, -0.0120849609375, -0.029266357421875, -0.03204345703125, 0.023590087890625, -0.0158843994140625, -0.51806640625, 0.01334381103515625, 0.0089874267578125, 0.0104217529296875, 0.047607421875, 0.00469207763671875, -0.0004456043243408203, -0.0104217529296875, 0.006526947021484375, -0.0267181396484375, -0.00974273681640625, -0.01983642578125, 0.022552490234375, -0.02337646484375, -0.0011034011840820312, -0.0176239013671875, 0.0007953643798828125, 0.016265869140625, -0.004314422607421875, -0.03216552734375, -0.01317596435546875, 0.1107177734375, 0.0017824172973632812, 0.0079803466796875, -0.0201416015625, -0.0002694129943847656, -0.00243377685546875, -0.0015926361083984375, 0.00975799560546875, -0.0251617431640625, 0.022674560546875, -0.0085296630859375, -0.003864288330078125, 0.0007314682006835938, 0.00527191162109375, -0.0092620849609375, 0.01080322265625, -0.009002685546875, 0.0009207725524902344, 0.0084228515625, -0.0042572021484375, -0.114990234375, -0.0035190582275390625, -0.003631591796875, 0.004383087158203125, 0.028350830078125, -0.03277587890625, 0.0167999267578125, -0.00695037841796875, 0.019683837890625, -0.032073974609375, 0.037322998046875, 0.0232391357421875, 0.005687713623046875, 0.01482391357421875, 0.06268310546875, 0.0026874542236328125, 0.01160430908203125, -0.02435302734375, -0.0009732246398925781, -0.0141448974609375, -0.01190948486328125, -0.1536865234375, -0.0160369873046875, 0.0044708251953125, 0.01050567626953125, -0.01499176025390625, 0.012969970703125, 0.0231781005859375, 0.0251617431640625, -0.0230255126953125, 0.01346588134765625, 0.0036334991455078125, -0.0028667449951171875, 0.0079193115234375, -0.008331298828125, -0.0010347366333007812, -0.041412353515625, 0.01519775390625, 0.0178070068359375, 0.00879669189453125, -0.003082275390625, 0.0011816024780273438, -0.006313323974609375, -0.01097869873046875, 0.00012230873107910156, 0.0233306884765625, 0.03857421875, -0.0843505859375, 0.0002446174621582031, 0.0137176513671875, -0.005168914794921875, -0.0034046173095703125, 0.007564544677734375, 0.02496337890625, 0.0145111083984375, -0.024993896484375, 0.00160980224609375, -0.005893707275390625, 0.016326904296875, 0.0078887939453125, 0.01444244384765625, 0.0095672607421875, 0.0232086181640625, 0.007526397705078125, -0.029296875, -0.00853729248046875, 0.0040283203125, -0.00275421142578125, -0.016510009765625, -0.00890350341796875, 0.015045166015625, 0.0018320083618164062, -0.0009179115295410156, 0.00010651350021362305, -0.0002665519714355469, -0.0158538818359375, -0.00540924072265625, -0.005035400390625, -0.0024566650390625, 0.0103759765625, -0.019012451171875, -0.020355224609375, -0.057861328125, 0.00910186767578125, 0.038421630859375, -0.004863739013671875, -0.037353515625, -0.00414276123046875, -0.01212310791015625, -0.00394439697265625, 0.04595947265625, 0.004131317138671875, -0.00978851318359375, -0.005565643310546875, 0.0011339187622070312, -0.006404876708984375, -0.03155517578125, -0.004146575927734375, 0.0170135498046875, -0.00792694091796875, 0.010498046875, 0.0035152435302734375, -0.0034580230712890625, 0.0157012939453125, -0.00893402099609375, 0.05810546875, -0.00342559814453125, -0.0079498291015625, 0.00736236572265625, 0.021270751953125, 0.026458740234375, 0.003570556640625, -0.0175933837890625, 0.0007085800170898438, -0.0221710205078125, -0.0092315673828125, -0.0226593017578125, -0.032440185546875, 0.0192718505859375, 0.03106689453125, -0.00714874267578125, 0.03192138671875, 0.005695343017578125, 0.01277923583984375, 0.004299163818359375, -4.863739013671875e-05, -0.014617919921875, -0.024810791015625, -0.0010318756103515625, -0.002918243408203125, 0.034423828125, -0.00492095947265625, 0.03570556640625, -0.145751953125, -0.027496337890625, 0.0276336669921875, 0.012420654296875, 0.02099609375, 0.0228424072265625, -0.0013647079467773438, 0.0119171142578125, -0.08782958984375, -0.0119781494140625, 0.0159454345703125, -0.004047393798828125, -0.01105499267578125, 0.0201873779296875, 0.0229644775390625, 0.0106201171875, 0.016876220703125, 0.039794921875, 0.0191192626953125, 0.036163330078125, 0.053985595703125, 0.006237030029296875, 0.0033206939697265625, -0.031158447265625, -0.0130157470703125, -0.0126190185546875, -0.0024738311767578125, 0.001232147216796875, -0.01064300537109375, -0.019744873046875, 0.00390625, 0.013397216796875, -0.0242156982421875, 0.01230621337890625, 0.005344390869140625, -0.029510498046875, 0.0103302001953125, 0.0013523101806640625, 0.022003173828125, -0.0308837890625, -0.0699462890625, 0.01861572265625, 0.00408172607421875, 0.0023975372314453125, 0.0081634521484375, 0.007305145263671875, 0.0184326171875, 0.0022182464599609375, -0.0016460418701171875, 0.034698486328125, 0.00319671630859375, 0.01175689697265625, -0.0286865234375, 0.0218048095703125, -0.00806427001953125, 0.00688934326171875, -0.0030155181884765625, -0.016632080078125, -0.01885986328125, 0.054229736328125, -0.00608062744140625, 0.004367828369140625, 0.0135040283203125, -0.00554656982421875, 0.0160064697265625, 0.0087890625, -0.006622314453125, 0.0015401840209960938, 0.0184173583984375, 0.0140838623046875, 0.002349853515625, -0.038116455078125, -0.030975341796875, 0.035980224609375, -0.0133819580078125, 0.01123809814453125, 0.0040283203125, 0.0019321441650390625, 0.00574493408203125, -0.00414276123046875, 0.016204833984375, -0.0364990234375, -0.0291900634765625, 0.01287841796875, -0.0034961700439453125, -0.01506805419921875, -0.00432586669921875, 0.020111083984375, 0.0124664306640625, -0.015716552734375, -0.01238250732421875, 0.0189361572265625, -0.052337646484375, 0.035980224609375, -0.015350341796875, 0.00897216796875, -0.0044708251953125, -0.0031890869140625, 0.0104217529296875, 0.022430419921875, -0.004985809326171875, 0.006595611572265625, -0.0109100341796875, 0.020599365234375, 0.01354217529296875, 0.0013370513916015625, -0.030517578125, 0.0128631591796875, -0.0258026123046875, 0.021759033203125, 0.00571441650390625, 0.00682830810546875, 0.0029010772705078125, 0.0148468017578125, 0.0026683807373046875, -0.0255584716796875, -0.0306854248046875, -0.004425048828125, 0.03826904296875, 0.019195556640625, -0.00958251953125, -0.002025604248046875, -0.00025916099548339844, 0.00015246868133544922, 0.00954437255859375, -0.022552490234375, 0.0070648193359375, 0.0255889892578125, 0.00811767578125, -0.05194091796875, -0.0292816162109375, -0.132080078125, -0.018524169921875, -0.01311492919921875, 0.000995635986328125, -0.006381988525390625, 0.0193023681640625, -0.00087738037109375, 0.0193939208984375, 0.0223236083984375, -0.0173492431640625, -0.00887298583984375, -0.03277587890625, 0.0024242401123046875, 0.0006976127624511719, 0.023284912109375, -0.00518798828125, -0.008575439453125, -0.00677490234375, 0.01123046875, -0.0262908935546875, 0.0031795501708984375, 0.0224456787109375, 0.01258087158203125, -0.026092529296875, -0.0002899169921875, 0.01412200927734375, 0.021697998046875, -0.017120361328125, -0.035125732421875, -0.0216217041015625, -0.01500701904296875, -0.0003578662872314453, 0.0123748779296875, -0.0204620361328125, -0.0047454833984375, -0.01226806640625, 0.0019702911376953125, 0.03607177734375, 2.3245811462402344e-05, -0.00447845458984375, 0.006809234619140625, -0.02349853515625, 0.016204833984375, -0.0019817352294921875, -0.01763916015625, -0.0241546630859375, -0.01678466796875, -0.0194549560546875, -0.0273895263671875, 0.00698089599609375, 0.013275146484375, -0.01219940185546875, -0.000270843505859375, 0.00768280029296875, 0.0286865234375, -0.0169219970703125, -0.0091094970703125, 0.01381683349609375, -0.00627899169921875, 0.0283203125, 0.0029850006103515625, -0.004241943359375, -0.01001739501953125, -0.00023937225341796875, -0.002010345458984375, -0.0003905296325683594, 0.003936767578125, -0.001865386962890625, 0.004734039306640625, -0.042236328125, -0.01142120361328125, 0.0254669189453125, -0.01473236083984375, -0.0144195556640625, -0.020263671875, -0.033966064453125, 0.042205810546875, 0.0023097991943359375, -0.0004801750183105469, 0.00637054443359375, -0.00772857666015625, 0.0100860595703125, 0.032196044921875, -0.006175994873046875, -0.00786590576171875, -0.022857666015625, 0.0217437744140625, -0.024810791015625, -0.00865936279296875, 0.006862640380859375, -0.0009012222290039062, 0.0132293701171875, 0.013153076171875, -0.0057220458984375, -0.00339508056640625, 0.00272369384765625, -0.0277099609375, -0.0075225830078125, 0.0283966064453125, -0.04071044921875, -0.0029754638671875, -0.032501220703125, -0.00902557373046875, -0.00676727294921875, -0.022552490234375, -0.01873779296875, -0.0243682861328125, 0.0171661376953125, 0.0223846435546875, -0.0294036865234375, 0.009857177734375, 0.0030231475830078125, -0.0026035308837890625, -0.0103607177734375, 0.017822265625, 0.004718780517578125, 0.024627685546875, -0.0175323486328125, 0.01373291015625, -0.028045654296875, -0.005504608154296875, -0.0236053466796875, 0.00046443939208984375, -0.023773193359375, -0.00788116455078125, 0.01355743408203125, -0.0027637481689453125, -0.0108184814453125, -0.0203857421875, -0.00690460205078125, 0.007289886474609375, 0.01248931884765625, 0.0175323486328125, 0.018829345703125, 0.00249481201171875, -0.007297515869140625, 0.0019321441650390625, -0.0308837890625, 0.02008056640625, -0.01239776611328125, -0.00859832763671875, -0.0002028942108154297, -0.0032672882080078125, -0.004726409912109375, 0.04010009765625, -0.09393310546875, -0.00921630859375, 0.0078582763671875, -0.01007843017578125, 0.01058197021484375, 0.00933837890625, -0.01207733154296875, 0.0273895263671875, -0.00215911865234375, -0.0028705596923828125, 0.0085296630859375, -0.023773193359375, -0.0026645660400390625, 0.00948333740234375, 0.02447509765625, -0.0238037109375, -0.024383544921875, 0.0050506591796875, -0.004913330078125, 0.00180816650390625, -1.1801719665527344e-05, -0.033935546875, 0.005931854248046875, 0.00824737548828125, 0.0665283203125, -0.0137176513671875, -0.0004448890686035156, -0.009124755859375, 0.0054473876953125, -0.0074310302734375, 0.01485443115234375, -0.0106201171875, 0.0262298583984375, 0.0292816162109375, -0.013580322265625, -0.0269317626953125, -0.005462646484375, 0.0194244384765625]\n",
            "896\n",
            "[0.021728515625, 0.00878143310546875, 0.0185699462890625, 0.0012693405151367188, -0.00183868408203125, -0.0101470947265625, -0.00710296630859375, 0.01136016845703125, -0.008514404296875, -0.002521514892578125, -0.04351806640625, 0.0157470703125, -0.0203094482421875, -0.01073455810546875, -0.001514434814453125, -0.00893402099609375, -0.12249755859375, 0.020111083984375, -0.007274627685546875, 0.0035724639892578125, -0.031768798828125, 0.02239990234375, -0.046112060546875, -0.017120361328125, -0.0037784576416015625, 0.0015277862548828125, -0.01068115234375, -0.007595062255859375, 0.00659942626953125, -0.0013437271118164062, 0.0246429443359375, -0.0173797607421875, 0.0225067138671875, 0.006969451904296875, -0.022857666015625, -0.006626129150390625, 0.015045166015625, -0.0133819580078125, -0.005855560302734375, 0.04296875, -0.020721435546875, 0.0210723876953125, 0.0006232261657714844, -0.0035991668701171875, 0.046417236328125, -0.0180816650390625, -0.0100860595703125, -0.0093231201171875, 0.01250457763671875, -0.02069091796875, 0.00930023193359375, 0.011810302734375, -0.01064300537109375, 0.0030670166015625, 0.00989532470703125, 0.0289306640625, -0.02288818359375, 0.0155181884765625, 0.01189422607421875, 0.007030487060546875, 0.0183868408203125, 0.045074462890625, 0.010528564453125, -0.0287628173828125, 0.1357421875, 0.0258026123046875, -0.00327301025390625, -0.0311431884765625, 0.0067596435546875, -0.030670166015625, 0.000949859619140625, 0.0194244384765625, 0.0229949951171875, -0.027099609375, 0.0162506103515625, -0.0160675048828125, 0.0036907196044921875, -0.016143798828125, 0.04541015625, -0.02740478515625, -0.0109405517578125, -0.0251617431640625, -0.0022907257080078125, -0.045379638671875, -0.01036834716796875, 0.0220794677734375, 0.0032711029052734375, -0.00688934326171875, 0.00936126708984375, -0.0007524490356445312, -0.0232391357421875, -0.032379150390625, 0.026031494140625, 0.0780029296875, 0.00439453125, 0.0269317626953125, 0.0185089111328125, 0.01006317138671875, 0.0025177001953125, 0.029296875, 0.0208282470703125, 0.0110015869140625, -0.005733489990234375, -0.0028362274169921875, 0.01273345947265625, 0.03167724609375, -0.0028076171875, 0.0019588470458984375, -0.023193359375, 0.01409149169921875, 0.003997802734375, 0.024078369140625, 0.04083251953125, 0.0138702392578125, -0.028533935546875, -0.0065155029296875, -0.0019359588623046875, 0.0190887451171875, -0.0030612945556640625, 0.0050506591796875, 0.0141143798828125, 0.00787353515625, -0.00667572021484375, 0.0247802734375, 0.00403594970703125, -0.00787353515625, 0.00860595703125, -0.00495147705078125, 0.01190185546875, -0.00496673583984375, -0.01224517822265625, -0.01515960693359375, -0.00196075439453125, -0.023162841796875, 0.0377197265625, 0.005458831787109375, -0.010406494140625, 0.015380859375, 0.01300811767578125, -0.0146026611328125, -7.212162017822266e-05, 0.0108642578125, -0.0280609130859375, -0.022552490234375, 0.01342010498046875, 0.01117706298828125, 0.0015716552734375, 0.00506591796875, 0.01153564453125, 0.0156097412109375, -0.049224853515625, -0.025360107421875, 0.002452850341796875, -0.0271453857421875, -0.031402587890625, -0.0273895263671875, 0.0064849853515625, -0.012176513671875, 0.0011501312255859375, -0.006473541259765625, -0.0198516845703125, 0.0166778564453125, 0.00994873046875, -0.016510009765625, 0.0109710693359375, 0.037261962890625, -0.0570068359375, 0.01247406005859375, -0.01467132568359375, -0.053924560546875, 0.034912109375, -0.01314544677734375, -0.005123138427734375, 0.0270233154296875, -0.00634002685546875, 0.026275634765625, 0.0286712646484375, -0.03790283203125, -0.00786590576171875, 0.023101806640625, 0.0114898681640625, -0.00049591064453125, 0.012054443359375, 0.0086517333984375, -0.0255126953125, -0.0264892578125, -0.018341064453125, -0.005153656005859375, 0.0091400146484375, 0.0020999908447265625, -0.0185699462890625, -0.0265045166015625, -0.03759765625, 0.00450897216796875, -0.025543212890625, -0.01155853271484375, 0.0135345458984375, -0.00115966796875, -0.0165557861328125, 0.0193023681640625, 0.00279998779296875, -0.03717041015625, 0.0022068023681640625, 0.0101318359375, -0.005054473876953125, 0.00891876220703125, -0.039581298828125, 0.0144195556640625, 0.1556396484375, -0.00838470458984375, 0.0017938613891601562, 0.0273284912109375, -0.00823974609375, -0.0379638671875, -0.012115478515625, 0.0086212158203125, -0.01259613037109375, -0.031707763671875, -0.0287017822265625, 0.00960540771484375, -0.00027632713317871094, 0.1588134765625, 0.001972198486328125, -0.030059814453125, 0.01560211181640625, -0.005023956298828125, 0.007175445556640625, 0.016937255859375, 0.01439666748046875, 0.03302001953125, 0.01068115234375, 0.0211334228515625, 0.0218963623046875, -0.000881195068359375, -0.0015811920166015625, -0.005023956298828125, 0.01256561279296875, -0.01593017578125, 0.0003917217254638672, 0.033050537109375, 0.0007376670837402344, 0.0016565322875976562, 0.01078033447265625, -0.01515960693359375, 0.01739501953125, 0.01540374755859375, -0.025299072265625, -0.03778076171875, -0.01320648193359375, -0.0172882080078125, 0.012481689453125, 6.61611557006836e-05, 0.0164642333984375, -0.002208709716796875, -0.0305023193359375, 0.01337432861328125, -0.01247406005859375, 0.0247802734375, -0.01052093505859375, 0.0211334228515625, -0.0221710205078125, -0.0274658203125, 0.03997802734375, -0.01763916015625, 0.01554107666015625, 0.00817108154296875, -0.0241851806640625, 0.028533935546875, -0.003997802734375, -0.007305145263671875, -0.00518035888671875, 0.01446533203125, 0.01531219482421875, -0.01435089111328125, 0.01006317138671875, 0.0180816650390625, -0.006145477294921875, -0.0021457672119140625, 0.0080413818359375, -0.0181884765625, 0.078857421875, 0.035980224609375, -0.01078033447265625, 0.001018524169921875, -0.014404296875, 0.0148773193359375, 0.01812744140625, 0.003173828125, -0.03167724609375, -0.037017822265625, -0.020263671875, 0.007598876953125, 0.05499267578125, 0.02374267578125, -0.0023860931396484375, 0.022613525390625, 0.0241546630859375, 0.00536346435546875, 0.0007572174072265625, -0.002002716064453125, 0.038970947265625, 0.0031528472900390625, 0.0006146430969238281, 0.022552490234375, -0.02716064453125, 0.0009317398071289062, 0.0017833709716796875, 0.0064239501953125, -0.0187530517578125, -0.025115966796875, -0.01381683349609375, -0.0007152557373046875, 0.025390625, -0.0215606689453125, 0.0204925537109375, 0.353515625, 0.00930023193359375, -0.0045623779296875, -0.0146636962890625, 0.01739501953125, -0.024505615234375, 0.032196044921875, -0.006694793701171875, 0.007144927978515625, 0.0162353515625, 0.0127410888671875, -0.0139923095703125, 0.0056915283203125, -0.006641387939453125, -0.01049041748046875, -0.00859832763671875, -0.0260009765625, -0.0126800537109375, -0.01959228515625, 0.0484619140625, 0.0270233154296875, 0.01090240478515625, 0.00800323486328125, 0.0047149658203125, 0.0058135986328125, 0.004322052001953125, -0.0037403106689453125, 0.02447509765625, -0.054656982421875, -0.01357269287109375, 0.0121612548828125, 0.0012493133544921875, 0.021759033203125, 0.00379180908203125, -0.041656494140625, -0.003971099853515625, 0.014068603515625, 0.00701904296875, -0.040191650390625, -0.0006885528564453125, 0.0003437995910644531, -0.003932952880859375, 0.0157623291015625, 0.01110076904296875, -0.0162811279296875, -0.01030731201171875, 0.00112152099609375, 0.0012483596801757812, 0.00853729248046875, -0.03619384765625, -0.005153656005859375, -0.2198486328125, -0.01222991943359375, 0.0217742919921875, -0.003108978271484375, -0.0125732421875, -0.021484375, -0.005558013916015625, 0.020416259765625, -0.022186279296875, -0.01544952392578125, 0.0107269287109375, 0.0301055908203125, 0.00301361083984375, -0.0031414031982421875, -0.0261688232421875, -0.004791259765625, -0.02130126953125, 0.0015687942504882812, 0.008392333984375, -0.034698486328125, -0.00348663330078125, 0.028717041015625, -0.018280029296875, -0.01503753662109375, -0.00945281982421875, 0.0231170654296875, -0.025238037109375, 0.003368377685546875, 0.00580596923828125, -0.00897216796875, -0.005817413330078125, -0.0288848876953125, 0.0271453857421875, 0.001987457275390625, -0.062744140625, 0.01096343994140625, -0.0003914833068847656, -0.051239013671875, -0.00865936279296875, -0.01995849609375, -0.001972198486328125, 0.015594482421875, -0.54833984375, 0.006717681884765625, -0.05535888671875, 0.0031871795654296875, -0.036865234375, -0.07080078125, 0.00905609130859375, -0.045074462890625, -0.0090789794921875, 0.004253387451171875, -0.0174713134765625, 0.020538330078125, -0.00766754150390625, 0.0229034423828125, -0.0111083984375, 0.0323486328125, 0.0391845703125, -0.012725830078125, -0.003021240234375, 0.014404296875, 0.0167083740234375, -0.00681304931640625, 0.0123291015625, 0.0229034423828125, -0.001255035400390625, 0.01312255859375, -0.010833740234375, -0.032958984375, 0.027740478515625, 0.0033779144287109375, 0.02545166015625, -0.0112152099609375, 0.0289764404296875, 0.0247955322265625, 0.00782012939453125, 0.00691986083984375, 0.0093536376953125, -0.005863189697265625, -0.009796142578125, 0.02655029296875, 0.04559326171875, -0.09429931640625, -0.017364501953125, -0.0225372314453125, -0.049774169921875, 0.0404052734375, 0.0006432533264160156, 0.00738525390625, -0.003902435302734375, 0.040008544921875, -0.0272369384765625, -0.005886077880859375, 0.0278778076171875, -0.0025806427001953125, -0.08819580078125, 0.00937652587890625, 0.0112457275390625, -0.00580596923828125, 0.017364501953125, 0.0007123947143554688, 0.0217742919921875, 0.011962890625, -0.10748291015625, -0.0036334991455078125, -0.0211181640625, 0.0233001708984375, -0.016510009765625, -0.033111572265625, 0.005496978759765625, 0.047149658203125, -0.00832366943359375, 0.00705718994140625, 0.0035648345947265625, 0.034210205078125, 0.01090240478515625, 0.005260467529296875, -0.009002685546875, -0.034210205078125, -0.0004336833953857422, 0.0232391357421875, 0.0039215087890625, 0.0028362274169921875, 0.00179290771484375, -0.02874755859375, 0.03167724609375, 0.00904083251953125, 0.015472412109375, -0.00269317626953125, -0.00624847412109375, -0.01432037353515625, 0.00797271728515625, -0.00585174560546875, 0.023101806640625, 0.0120697021484375, 0.039947509765625, -0.0143280029296875, 0.054290771484375, 0.0184478759765625, -0.00519561767578125, 0.0030612945556640625, 0.0009984970092773438, 0.0279541015625, -0.005153656005859375, 0.00799560546875, -0.00267791748046875, 0.0034847259521484375, -0.0061492919921875, -0.00174713134765625, 0.028472900390625, 0.005298614501953125, 0.01479339599609375, 0.005138397216796875, 0.0088043212890625, 0.01409149169921875, -0.0245819091796875, 0.00687408447265625, -0.0196685791015625, -0.007122039794921875, 0.0296630859375, 0.0119171142578125, -0.0013799667358398438, 0.004138946533203125, 0.002696990966796875, -0.01142120361328125, -0.004055023193359375, 0.0179901123046875, 0.035858154296875, 0.00415802001953125, 0.046539306640625, 0.01459503173828125, 0.01161956787109375, 0.0794677734375, -0.0029296875, -0.005817413330078125, 0.0255279541015625, -0.0037403106689453125, 0.033966064453125, -0.0212554931640625, -0.0016775131225585938, 0.0088043212890625, -0.022003173828125, -0.00577545166015625, -0.01195526123046875, 0.01079559326171875, -0.01055145263671875, -0.0023593902587890625, 0.04327392578125, -0.01496124267578125, 0.029693603515625, -0.001102447509765625, 0.005123138427734375, 0.0202789306640625, -0.012176513671875, -0.0015592575073242188, 0.00994873046875, -0.008148193359375, -0.026153564453125, 0.015594482421875, 0.021331787109375, 0.00038552284240722656, 0.02362060546875, 0.023834228515625, 0.057952880859375, -0.01085662841796875, 0.00873565673828125, -0.01126861572265625, -0.007904052734375, 0.024322509765625, 0.0009083747863769531, -0.004383087158203125, -0.01312255859375, 0.019500732421875, 0.028778076171875, 0.01166534423828125, 0.021453857421875, 0.0188140869140625, 0.004467010498046875, 0.003215789794921875, 0.020721435546875, -0.0149383544921875, -0.0162506103515625, -0.003604888916015625, -0.0204620361328125, -0.00859832763671875, 0.0216827392578125, 0.0200042724609375, -0.0182037353515625, -0.00966644287109375, 0.044647216796875, 0.0017681121826171875, 0.028106689453125, -0.002819061279296875, 0.00609588623046875, 0.07293701171875, 0.02252197265625, 0.00836181640625, 0.01152801513671875, -0.022857666015625, 0.035369873046875, 0.01415252685546875, 0.00020897388458251953, 0.0263824462890625, -0.01050567626953125, -0.014739990234375, 0.00408172607421875, 0.006500244140625, 0.0110015869140625, -0.01617431640625, -0.0002658367156982422, 0.008575439453125, -0.0005350112915039062, -0.01556396484375, 0.0251922607421875, 0.01226043701171875, -0.06365966796875, 0.0007767677307128906, 0.00408935546875, -0.043365478515625, 0.0117950439453125, 0.01551055908203125, 0.007843017578125, -0.005176544189453125, -0.0255126953125, 0.005039215087890625, 0.00542449951171875, 0.0241851806640625, 0.01457977294921875, 0.0117034912109375, 0.0203399658203125, 0.01026153564453125, -0.00021648406982421875, -0.0251617431640625, -0.014678955078125, 0.0029144287109375, -0.00803375244140625, -0.0017690658569335938, 0.0185089111328125, 0.00975799560546875, -0.00797271728515625, -0.0123138427734375, -0.032867431640625, -0.00464630126953125, 0.01221466064453125, -0.0228424072265625, -0.0005488395690917969, -0.0195465087890625, -0.005092620849609375, 0.0792236328125, -0.024993896484375, 0.0210418701171875, 0.0023040771484375, -0.004436492919921875, -0.0237884521484375, -0.002902984619140625, 0.038482666015625, -0.046875, -0.0120391845703125, 0.023040771484375, -0.023681640625, -0.0109710693359375, -0.046844482421875, -0.0018949508666992188, 0.00734710693359375, 0.0362548828125, -0.0222930908203125, -0.02618408203125, 0.01326751708984375, 0.0101318359375, -0.02581787109375, -0.023223876953125, -0.006134033203125, -0.0300750732421875, -0.025726318359375, 0.01264190673828125, 0.0177764892578125, 0.0031871795654296875, 0.00970458984375, 0.009033203125, -0.0005049705505371094, 0.01229095458984375, -0.0076141357421875, -0.00890350341796875, -0.03240966796875, 0.00812530517578125, 0.01189422607421875, 0.032379150390625, -0.0171966552734375, 0.009552001953125, -0.000659942626953125, 0.02496337890625, -0.0054931640625, -0.0145721435546875, 0.0016574859619140625, -0.006435394287109375, -0.0445556640625, 0.0266265869140625, 0.016632080078125, 0.02996826171875, 0.004032135009765625, 0.027130126953125, -0.0131988525390625, -0.0205230712890625, 0.01296234130859375, -0.04071044921875, -0.0333251953125, -0.06591796875, 0.024658203125, -0.00255584716796875, 0.0171966552734375, 0.0198211669921875, 0.04443359375, -0.036407470703125, 0.0212860107421875, -0.004749298095703125, -0.01666259765625, -0.0311126708984375, -0.003337860107421875, 0.0179595947265625, 0.11444091796875, 0.034820556640625, -0.008514404296875, 0.025726318359375, -0.0173187255859375, -0.01146697998046875, -0.038055419921875, 0.0028076171875, 0.0252685546875, -0.006656646728515625, -0.033660888671875, 0.01206207275390625, -0.0125885009765625, -0.01284027099609375, -0.01861572265625, -0.019317626953125, 0.0178375244140625, -0.021942138671875, 0.0092315673828125, 0.036651611328125, -0.00921630859375, -0.0010986328125, -0.0156402587890625, 0.0255126953125, -0.0293731689453125, -0.050811767578125, -0.005401611328125, 0.0198516845703125, 0.0207977294921875, 0.0013427734375, 0.00997161865234375, -0.0310211181640625, -0.00850677490234375, -0.0499267578125, 0.03338623046875, 0.0012569427490234375, 0.006526947021484375, -0.01465606689453125, -0.00920867919921875, 0.02667236328125, -0.020660400390625, 0.033599853515625, -0.0069122314453125, 0.0010356903076171875, 0.01580810546875, 0.006191253662109375, 0.0060272216796875, -0.0192108154296875, -0.025299072265625, -0.04180908203125, 0.0009021759033203125, -0.00039887428283691406, -0.030242919921875, 0.005771636962890625, 0.00994873046875, 0.017059326171875, 0.004711151123046875, -0.00833892822265625, 0.0168609619140625, -0.0031070709228515625, 0.00514984130859375, -0.034759521484375, -0.0253753662109375, 0.01593017578125, -0.0684814453125, -0.003017425537109375, 0.004451751708984375, -0.0004558563232421875, 0.043975830078125, 0.0484619140625, 0.007343292236328125, -0.021484375, -0.0255279541015625, 0.02740478515625, 0.00409698486328125, -0.03472900390625, 0.01117706298828125, 0.017913818359375, 0.0078887939453125, -7.700920104980469e-05, 0.03515625, -0.00676727294921875, 0.042327880859375, -0.01032257080078125, 0.0121612548828125, 0.0179595947265625, -0.027252197265625, -0.005687713623046875, -0.01226043701171875, 0.007122039794921875, -0.0256195068359375, 0.0311279296875, -0.007068634033203125, 0.00067901611328125, -0.002410888671875, -0.006683349609375, -0.0284271240234375, 0.0233917236328125, -0.0153656005859375, -0.0281524658203125, 0.002727508544921875, 0.01036834716796875, -0.02886962890625, 0.0263671875, -0.0075836181640625, -0.004001617431640625, -0.00931549072265625, 0.0030612945556640625, -0.03106689453125, -0.02459716796875, -0.0106658935546875, 0.00970458984375, 0.01641845703125, 0.04937744140625, 0.033447265625, -0.0020465850830078125, 0.005710601806640625, 0.006519317626953125, -0.026214599609375, -0.03753662109375, 0.038970947265625, -0.009613037109375, 0.033416748046875, -0.00862884521484375, -0.007175445556640625, 0.041656494140625, -0.01047515869140625, -0.0006074905395507812, 0.0008263587951660156, -0.0012111663818359375, 0.01398468017578125, -0.00563812255859375, -0.06939697265625, -0.0166778564453125, 0.025299072265625, 0.0179443359375, 0.032989501953125, 0.016448974609375, -0.0012445449829101562, -0.004547119140625, -0.0092926025390625, 0.0121307373046875, -0.002162933349609375, -0.024810791015625, -0.0182037353515625, 0.0330810546875, 0.01378631591796875, -0.00872802734375, -0.01503753662109375, 0.005626678466796875, 0.0024871826171875, -0.0260162353515625, -0.0128326416015625, -0.0196685791015625, -0.00574493408203125, -0.003047943115234375, 0.033599853515625, 0.00921630859375, 0.01548004150390625, -0.00605010986328125, -0.0205230712890625, -0.022003173828125, -0.017059326171875, -0.007755279541015625, -0.0164337158203125, -0.02606201171875, -0.01114654541015625, 0.004322052001953125, -0.0171051025390625, -0.0009336471557617188]\n",
            "896\n",
            "[-0.035247802734375, 0.01947021484375, 0.0177154541015625, 0.0166168212890625, 0.0017833709716796875, -0.000888824462890625, -0.01373291015625, 0.000885009765625, 0.010284423828125, -0.018585205078125, -0.031005859375, -0.0029697418212890625, 0.0170440673828125, 0.01305389404296875, 0.023529052734375, -0.0038051605224609375, -0.222900390625, -0.01439666748046875, -0.0304107666015625, 0.01462554931640625, -0.019317626953125, 0.0178985595703125, -0.007640838623046875, 0.0196380615234375, -0.01837158203125, -0.0673828125, -0.005828857421875, -0.00033926963806152344, 0.0036830902099609375, 0.0097503662109375, -0.00504302978515625, -0.0274658203125, -0.0137939453125, -0.005908966064453125, -0.002307891845703125, -0.000705718994140625, -0.004444122314453125, 0.0217742919921875, -0.01483917236328125, -0.0172119140625, 0.0623779296875, 0.00888824462890625, 0.03887939453125, 0.0084228515625, -0.0117950439453125, 0.0106353759765625, -0.0186767578125, 0.0105438232421875, -0.0041961669921875, -0.0110015869140625, -0.02294921875, -0.000904083251953125, -0.0109100341796875, -0.0018720626831054688, 0.0274200439453125, 0.00569915771484375, -0.007076263427734375, 0.017181396484375, 0.025482177734375, -0.0020618438720703125, 0.006755828857421875, 0.04461669921875, -0.0082550048828125, 0.0004127025604248047, 0.01049041748046875, 0.0035915374755859375, 0.016387939453125, -0.0384521484375, 0.0016689300537109375, -0.038818359375, -0.01355743408203125, 0.0007328987121582031, 0.00998687744140625, 0.00797271728515625, -0.002140045166015625, -0.00311279296875, 0.0191497802734375, 0.01102447509765625, -0.003131866455078125, 0.0338134765625, 0.00597381591796875, 0.032073974609375, 0.0032520294189453125, -0.006023406982421875, -0.0188751220703125, 0.0094451904296875, 0.01477813720703125, 0.0091705322265625, 0.004604339599609375, -0.0293426513671875, 0.012420654296875, 0.0286865234375, 0.004329681396484375, 0.1126708984375, 0.00754547119140625, -0.019287109375, 0.01328277587890625, -0.00860595703125, -0.0095367431640625, 0.061279296875, 0.0299835205078125, 0.0013685226440429688, -0.007053375244140625, -0.0155792236328125, 0.0146484375, -0.0300140380859375, 0.0013494491577148438, 0.005855560302734375, -0.01239776611328125, -0.00681304931640625, 0.0009479522705078125, -0.0098419189453125, 0.0170745849609375, 0.00800323486328125, -0.00775909423828125, 0.0033016204833984375, -0.0090484619140625, 0.0197296142578125, 0.0185394287109375, 0.00449371337890625, 0.04132080078125, -0.01329803466796875, 0.008819580078125, 0.00678253173828125, 0.00646209716796875, -0.0060272216796875, 0.01242828369140625, -0.002971649169921875, 0.00867462158203125, 0.006046295166015625, 0.040740966796875, 0.01438140869140625, 0.034912109375, 0.007381439208984375, 0.0087890625, -0.01053619384765625, -0.019989013671875, 0.0231475830078125, -0.0254058837890625, 0.00679779052734375, -0.0031337738037109375, -0.01430511474609375, -0.02752685546875, -0.00748443603515625, -0.01617431640625, -0.004596710205078125, 0.05120849609375, -0.0012426376342773438, 0.01898193359375, 0.0074462890625, 0.0085296630859375, 0.00856781005859375, -0.0224761962890625, -0.0227203369140625, 0.0002193450927734375, -0.0013904571533203125, -0.06884765625, -0.0181121826171875, 0.048187255859375, 0.028564453125, 0.010650634765625, -0.01189422607421875, -0.0084381103515625, 0.0093841552734375, -0.018951416015625, -0.0006384849548339844, -0.0290374755859375, -0.008209228515625, -0.007720947265625, 5.2809715270996094e-05, 0.0131378173828125, 0.0141448974609375, 0.01038360595703125, 0.03570556640625, 0.01111602783203125, 0.00823974609375, -0.0537109375, -0.01505279541015625, -0.00858306884765625, -0.0016536712646484375, 0.020843505859375, -0.00028252601623535156, -0.0087127685546875, -0.011505126953125, -0.0218505859375, 0.0007510185241699219, -0.0278167724609375, 0.0088043212890625, 0.0279083251953125, -0.004375457763671875, -0.006893157958984375, 0.00653076171875, -0.0254058837890625, -0.0270538330078125, -0.004974365234375, 0.0210113525390625, -0.0019664764404296875, -0.005519866943359375, 0.0693359375, -0.01418304443359375, -0.01363372802734375, -0.05230712890625, 0.00958251953125, 0.00121307373046875, 0.0322265625, 0.01009368896484375, -0.00519561767578125, -0.03900146484375, 0.22705078125, -0.01233673095703125, 0.0191650390625, -0.00409698486328125, -0.001216888427734375, 0.002712249755859375, 0.011016845703125, 0.005817413330078125, 0.01404571533203125, -0.0002143383026123047, 2.3066997528076172e-05, 0.00205230712890625, -0.006412506103515625, 0.0972900390625, -0.00555419921875, 0.0361328125, 0.0007491111755371094, -0.004909515380859375, -0.01812744140625, -0.001983642578125, 0.02508544921875, -0.0006117820739746094, 0.0179290771484375, 0.01282501220703125, -0.003208160400390625, -0.024932861328125, 0.0298004150390625, -0.003749847412109375, 0.0154876708984375, -0.01142120361328125, -0.005619049072265625, 0.001819610595703125, -0.031219482421875, 0.0200653076171875, -0.01375579833984375, -0.018707275390625, 0.0010843276977539062, 0.0036449432373046875, -0.0171356201171875, -0.026153564453125, 0.0058746337890625, -0.005298614501953125, 0.0266265869140625, 0.021453857421875, 0.01261138916015625, -0.013824462890625, -0.00881195068359375, 0.0016984939575195312, 0.005084991455078125, 0.007617950439453125, -0.0032958984375, -0.0024967193603515625, -0.0059814453125, -0.01873779296875, 0.02972412109375, 0.0168914794921875, -0.000461578369140625, 0.0077362060546875, -0.113037109375, 0.022491455078125, -0.00940704345703125, -0.00807952880859375, 0.002437591552734375, 0.01325225830078125, 0.0163116455078125, -0.0249786376953125, -0.0024166107177734375, 0.0085906982421875, 0.01336669921875, 0.0100555419921875, 0.01739501953125, 0.003421783447265625, -0.02679443359375, 0.0006384849548339844, 0.005954742431640625, -0.0089569091796875, 0.0162506103515625, -0.0093994140625, -0.01482391357421875, 0.00907135009765625, -0.01447296142578125, -0.02056884765625, -0.01486968994140625, 0.01419830322265625, 0.00978851318359375, 0.00693511962890625, 0.01287841796875, -0.0163421630859375, 0.0252685546875, 0.003692626953125, 0.0007376670837402344, 0.00888824462890625, 0.09259033203125, -0.015411376953125, -0.01385498046875, 0.009185791015625, -0.0163116455078125, -0.008056640625, -0.0092926025390625, 0.00962066650390625, -0.0008025169372558594, -0.00962066650390625, 0.003448486328125, -0.0086212158203125, 0.00835418701171875, -0.0006079673767089844, 0.0222625732421875, 0.294921875, 0.014739990234375, 0.0003476142883300781, 0.003261566162109375, -0.00057220458984375, 0.0240478515625, -0.0174407958984375, -0.0190582275390625, -0.010009765625, -0.0164947509765625, -0.01450347900390625, -0.009918212890625, 0.0017099380493164062, -0.031402587890625, 0.0281829833984375, -0.0306549072265625, 0.005176544189453125, 0.004589080810546875, 0.0013427734375, -0.04931640625, 0.01270294189453125, 0.01493072509765625, -0.002811431884765625, -0.0008039474487304688, 0.00739288330078125, 0.052154541015625, 0.004077911376953125, 0.048187255859375, 0.09344482421875, -0.037689208984375, -0.004680633544921875, -0.0173492431640625, -0.008453369140625, -0.0208282470703125, 0.0201263427734375, -0.01666259765625, -0.0035419464111328125, 0.0254974365234375, -0.00559234619140625, 0.01221466064453125, -0.036346435546875, -0.0034942626953125, -0.01849365234375, 0.00545501708984375, -0.0131683349609375, 0.01267242431640625, 0.0151214599609375, -0.0045623779296875, 0.029296875, 0.035888671875, 0.01568603515625, -0.1102294921875, -0.00353240966796875, -0.0011148452758789062, 0.01012420654296875, 0.00988006591796875, -0.031463623046875, -0.008453369140625, 0.005039215087890625, -0.020111083984375, 0.0172119140625, -0.010528564453125, 0.0142059326171875, -0.003021240234375, -0.0035037994384765625, -0.00930023193359375, -0.003002166748046875, -0.0070037841796875, 0.00756072998046875, 0.0145263671875, 0.0239105224609375, 0.0215301513671875, 0.021636962890625, -0.04693603515625, -0.020751953125, -0.0799560546875, 0.00910186767578125, 0.041961669921875, -0.013916015625, 0.0206146240234375, -0.003143310546875, -9.238719940185547e-06, -0.0180816650390625, 0.01523590087890625, 0.037689208984375, -0.057403564453125, -0.01500701904296875, -0.006114959716796875, 0.0161895751953125, 0.006992340087890625, 0.01153564453125, -0.03533935546875, -0.0029544830322265625, -0.5966796875, -0.002712249755859375, -0.013702392578125, 0.007358551025390625, -0.0628662109375, -0.03619384765625, -0.0178985595703125, -0.01496124267578125, 0.00710296630859375, -0.00954437255859375, -0.0118408203125, 0.0038604736328125, 0.002040863037109375, 0.0245208740234375, 0.0017385482788085938, -0.0186309814453125, -0.02972412109375, 0.01221466064453125, -0.0184783935546875, -0.045379638671875, -0.0242919921875, 0.0169677734375, 0.0084991455078125, 0.0019435882568359375, -0.0088958740234375, -0.00463104248046875, -0.00566864013671875, -0.026702880859375, -0.0240631103515625, 0.019317626953125, 0.01898193359375, -0.01300048828125, 0.01934814453125, -0.016326904296875, 0.014007568359375, -0.00836181640625, 0.01071929931640625, 0.0023593902587890625, 0.001995086669921875, -0.0269317626953125, 0.0016422271728515625, -0.0673828125, 0.0283660888671875, 0.0226287841796875, -0.01468658447265625, -0.0160064697265625, -0.008758544921875, 0.0014142990112304688, -0.0049591064453125, -0.0017328262329101562, -0.01149749755859375, 0.015350341796875, -0.0011701583862304688, 0.01447296142578125, 0.25439453125, -0.023834228515625, 0.000286102294921875, 0.0156707763671875, 0.0174560546875, 0.0009169578552246094, 0.022705078125, -0.031585693359375, 0.0179901123046875, -0.0106658935546875, -0.005237579345703125, 0.036346435546875, -0.0049285888671875, -0.006984710693359375, -0.005748748779296875, 0.004772186279296875, 0.01332855224609375, -0.0038471221923828125, 0.0126495361328125, -0.0009164810180664062, 0.0007472038269042969, -0.0034313201904296875, -0.0057373046875, 0.020477294921875, 0.034393310546875, -0.0302734375, -0.00092315673828125, -0.0009298324584960938, 0.0010128021240234375, -0.0083770751953125, -0.0013408660888671875, -0.0160980224609375, 0.006771087646484375, -0.006683349609375, -0.021209716796875, 0.01190185546875, -0.01328277587890625, 0.0224151611328125, 0.0343017578125, 0.0136260986328125, -0.01024627685546875, 0.0007076263427734375, -0.01312255859375, 0.01276397705078125, 0.0093994140625, 0.0054473876953125, -0.002162933349609375, 0.0006928443908691406, -0.0175323486328125, 0.0306549072265625, -0.025604248046875, -0.07818603515625, -0.0019683837890625, 0.0153961181640625, 0.0250701904296875, -0.0159759521484375, -0.01171112060546875, 0.023406982421875, 0.0141448974609375, -0.0139923095703125, -0.005290985107421875, -0.00930023193359375, -0.0014657974243164062, 0.0096435546875, 0.0112152099609375, 0.0029277801513671875, -0.007781982421875, -0.0234222412109375, 0.013092041015625, 0.006084442138671875, -0.0080718994140625, 0.0416259765625, 0.010894775390625, -0.010162353515625, -0.02545166015625, -0.00141143798828125, 0.01279449462890625, -0.00018203258514404297, -0.0154876708984375, -0.0078125, 0.0024890899658203125, 0.031402587890625, -0.0215606689453125, -0.00989532470703125, -0.006366729736328125, -0.00426483154296875, 0.006320953369140625, -0.0194549560546875, 0.01383209228515625, -0.0135345458984375, 0.00827789306640625, -0.01190185546875, 0.022857666015625, -0.028656005859375, 0.04254150390625, 0.004199981689453125, 0.01447296142578125, 0.050506591796875, -0.0041046142578125, -0.0020389556884765625, -0.015777587890625, -0.00701904296875, 0.0247802734375, 0.00994873046875, 0.019378662109375, -0.0234527587890625, 0.0103302001953125, 0.0177459716796875, 0.06390380859375, 0.0015735626220703125, 0.00931549072265625, -0.00638580322265625, 0.00135040283203125, -0.00025200843811035156, -0.0067291259765625, -0.0162811279296875, -0.0235748291015625, 0.0171966552734375, 0.00870513916015625, -0.0088958740234375, -0.039398193359375, 0.0067596435546875, -0.00640106201171875, 0.01727294921875, 0.007579803466796875, -0.0014028549194335938, -0.0300445556640625, -0.0159759521484375, 0.029510498046875, -0.00873565673828125, -0.01148223876953125, -0.015594482421875, -0.00466156005859375, 0.0046234130859375, 0.032562255859375, 0.023162841796875, -0.01462554931640625, -0.004894256591796875, 0.0077362060546875, 0.08453369140625, -0.00843048095703125, -0.009002685546875, 0.016143798828125, 0.003154754638671875, 0.015228271484375, 0.00997161865234375, -0.00948333740234375, 0.0114288330078125, 0.01104736328125, -0.025543212890625, -0.0162353515625, 0.0252685546875, -0.012298583984375, -0.0205078125, -0.003429412841796875, -0.00579833984375, 0.01275634765625, -0.0025768280029296875, -0.0295562744140625, -0.0216064453125, -0.03436279296875, -0.0135345458984375, 0.0043792724609375, 0.01253509521484375, 0.020538330078125, -0.007472991943359375, 0.01812744140625, 0.00936126708984375, 0.0048980712890625, -0.0192108154296875, 0.016876220703125, -0.01296234130859375, -0.0198822021484375, -0.0036869049072265625, 0.004711151123046875, -5.7220458984375e-05, 0.0285186767578125, -0.0232391357421875, -0.01224517822265625, 0.0072021484375, -0.006809234619140625, -0.023193359375, -0.0115509033203125, -0.022613525390625, 0.0171661376953125, 0.0211639404296875, -0.0640869140625, 0.0250396728515625, -0.01262664794921875, 0.01212310791015625, -0.00826263427734375, -0.0113525390625, 0.006153106689453125, 0.01419830322265625, 0.00925445556640625, -0.0261688232421875, 0.01447296142578125, -0.01641845703125, -0.00374603271484375, 0.0081787109375, -0.0257568359375, 0.0028438568115234375, -0.017974853515625, -0.008819580078125, -0.00557708740234375, -0.0219268798828125, 0.0074005126953125, -0.001407623291015625, 0.005168914794921875, -0.01375579833984375, -0.0037784576416015625, 0.01544189453125, -0.002330780029296875, 0.0271148681640625, -0.009796142578125, -0.0247344970703125, -0.024810791015625, -0.01279449462890625, -0.01934814453125, -0.0258026123046875, 0.0157470703125, 0.00614166259765625, -0.01708984375, 1.7344951629638672e-05, -0.0518798828125, -0.007160186767578125, -0.0170440673828125, -0.00295257568359375, -0.006008148193359375, -0.00879669189453125, -0.01467132568359375, 0.007080078125, -0.01242828369140625, 0.002628326416015625, -0.02520751953125, -0.030517578125, -0.00775146484375, 0.01447296142578125, -0.01605224609375, 0.00518035888671875, -0.006622314453125, 0.0262603759765625, 0.010223388671875, -0.006626129150390625, 0.018463134765625, 0.027313232421875, -0.00518035888671875, 0.0303802490234375, -0.01174163818359375, -0.00904083251953125, 0.0194244384765625, -0.0655517578125, -0.0015239715576171875, -0.0213623046875, -0.006587982177734375, 0.0050048828125, -0.0210418701171875, 0.0100555419921875, 0.0287628173828125, 0.004177093505859375, 0.005889892578125, 0.0034694671630859375, -0.003231048583984375, 0.0088043212890625, 0.0216522216796875, 0.0307769775390625, -0.0212249755859375, 0.0225830078125, 0.0136260986328125, 0.002033233642578125, 0.003265380859375, 0.01168060302734375, 0.01239776611328125, -0.01340484619140625, 0.015106201171875, 0.037139892578125, -0.003978729248046875, -0.01445770263671875, -0.0217742919921875, -0.0758056640625, 0.01495361328125, -0.03509521484375, 0.00872039794921875, 0.0144805908203125, -0.0235443115234375, 0.0014743804931640625, -0.01690673828125, 0.014190673828125, 0.00890350341796875, -0.00971221923828125, -0.007694244384765625, 0.01214599609375, 0.01500701904296875, -0.01538848876953125, -0.00943756103515625, -0.00899505615234375, -0.0166015625, -0.014892578125, 0.0147705078125, -0.006389617919921875, -0.01519012451171875, 0.00832366943359375, 0.0006442070007324219, -0.054412841796875, -0.0178680419921875, -0.007068634033203125, -0.0161895751953125, 0.0070037841796875, -0.00878143310546875, -0.006160736083984375, -0.0154876708984375, -0.01261138916015625, 0.01212310791015625, -0.0153961181640625, -0.006427764892578125, -0.01538848876953125, -0.001617431640625, -0.0279083251953125, -0.01410675048828125, 0.018707275390625, -0.01308441162109375, -0.0225372314453125, 0.020263671875, -0.01332855224609375, 0.004703521728515625, -9.47713851928711e-06, 0.032379150390625, 0.034454345703125, -0.0102691650390625, 0.0050048828125, 0.0116729736328125, -0.0084381103515625, -0.0031223297119140625, 0.00452423095703125, 0.0031185150146484375, 0.004425048828125, 0.0018224716186523438, -0.0294189453125, 0.0105743408203125, 0.01192474365234375, 0.0070037841796875, -0.02569580078125, -0.00238800048828125, 0.00949859619140625, -0.0026683807373046875, -0.01145172119140625, -0.051239013671875, -0.00771331787109375, -0.0067901611328125, 0.0242919921875, 0.0165252685546875, 0.0006494522094726562, 0.00843048095703125, -0.023651123046875, 0.003009796142578125, -0.02154541015625, -0.021820068359375, 0.0017881393432617188, -0.0171051025390625, 0.053253173828125, -0.0023555755615234375, 0.005268096923828125, -0.048431396484375, 0.0025081634521484375, 0.0019235610961914062, -0.0054931640625, 0.0030364990234375, 0.006343841552734375, 0.0136566162109375, 0.0181121826171875, -0.0458984375, 0.0101776123046875, -0.0184173583984375, -0.006778717041015625, -0.00705718994140625, -0.01385498046875, -0.0011110305786132812, -0.00017249584197998047, -0.0176849365234375, -0.01247406005859375, -0.00868988037109375, -0.0026092529296875, -0.0135498046875, 0.01500701904296875, 0.01213836669921875, -0.005214691162109375, -0.00675201416015625, 0.0167388916015625, 0.00803375244140625, 0.0272979736328125, 0.0142364501953125, -0.0111541748046875, -0.002323150634765625, 0.01702880859375, 0.0218048095703125, 0.0176544189453125, -0.0201568603515625, -0.0272369384765625, -0.024505615234375, 0.009246826171875, 0.013214111328125, 0.0046234130859375, 0.00640106201171875, 0.01081085205078125, 0.007720947265625, -0.0165557861328125, 0.000217437744140625, 0.0141448974609375, -0.00018906593322753906, 0.01105499267578125, 0.0177001953125, -0.0382080078125, -0.0065765380859375, 0.0167694091796875, 0.0009908676147460938, -0.0295562744140625, -0.00013506412506103516, 0.0201568603515625, -0.005462646484375, -0.001888275146484375, -0.03875732421875, -0.00366973876953125, 0.005931854248046875, 0.010009765625, -0.0119171142578125, 0.0032253265380859375, -0.0176849365234375, -0.034088134765625, 0.023223876953125, -0.0030803680419921875, 0.0207977294921875, -0.01568603515625, -0.005802154541015625, -0.01117706298828125]\n",
            "896\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "lrhQAtDuHiKY"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}