Portability_3_hop.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v_cGEnqs9An4",
        "outputId": "c6121ec5-58c0-46e8-8d6b-ce16d1f2997d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting datasets\n",
            "  Downloading datasets-2.18.0-py3-none-any.whl (510 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m510.5/510.5 kB\u001b[0m \u001b[31m3.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting accelerate\n",
            "  Downloading accelerate-0.29.2-py3-none-any.whl (297 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m297.4/297.4 kB\u001b[0m \u001b[31m23.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from datasets) (3.13.4)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from datasets) (1.25.2)\n",
            "Requirement already satisfied: pyarrow>=12.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (14.0.2)\n",
            "Requirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.10/dist-packages (from datasets) (0.6)\n",
            "Collecting dill<0.3.9,>=0.3.0 (from datasets)\n",
            "  Downloading dill-0.3.8-py3-none-any.whl (116 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m14.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (2.0.3)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (4.66.2)\n",
            "Collecting xxhash (from datasets)\n",
            "  Downloading xxhash-3.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting multiprocess (from datasets)\n",
            "  Downloading multiprocess-0.70.16-py310-none-any.whl (134 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m11.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: fsspec[http]<=2024.2.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2023.6.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.9.3)\n",
            "Requirement already satisfied: huggingface-hub>=0.19.4 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.20.3)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets) (24.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (6.0.1)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate) (5.9.5)\n",
            "Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (2.2.1+cu121)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from accelerate) (0.4.2)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->datasets) (4.11.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2024.2.2)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.1.3)\n",
            "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch>=1.10.0->accelerate)\n",
            "  Using cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
            "Collecting nvidia-cuda-runtime-cu12==12.1.105 (from torch>=1.10.0->accelerate)\n",
            "  Using cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
            "Collecting nvidia-cuda-cupti-cu12==12.1.105 (from torch>=1.10.0->accelerate)\n",
            "  Using cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
            "Collecting nvidia-cudnn-cu12==8.9.2.26 (from torch>=1.10.0->accelerate)\n",
            "  Using cached nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n",
            "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch>=1.10.0->accelerate)\n",
            "  Using cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
            "Collecting nvidia-cufft-cu12==11.0.2.54 (from torch>=1.10.0->accelerate)\n",
            "  Using cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
            "Collecting nvidia-curand-cu12==10.3.2.106 (from torch>=1.10.0->accelerate)\n",
            "  Using cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
            "Collecting nvidia-cusolver-cu12==11.4.5.107 (from torch>=1.10.0->accelerate)\n",
            "  Using cached nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
            "Collecting nvidia-cusparse-cu12==12.1.0.106 (from torch>=1.10.0->accelerate)\n",
            "  Using cached nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
            "Collecting nvidia-nccl-cu12==2.19.3 (from torch>=1.10.0->accelerate)\n",
            "  Using cached nvidia_nccl_cu12-2.19.3-py3-none-manylinux1_x86_64.whl (166.0 MB)\n",
            "Collecting nvidia-nvtx-cu12==12.1.105 (from torch>=1.10.0->accelerate)\n",
            "  Using cached nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
            "Requirement already satisfied: triton==2.2.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2.2.0)\n",
            "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.10.0->accelerate)\n",
            "  Using cached nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl (21.1 MB)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2023.4)\n",
            "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2024.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.5)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\n",
            "Installing collected packages: xxhash, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, dill, nvidia-cusparse-cu12, nvidia-cudnn-cu12, multiprocess, nvidia-cusolver-cu12, datasets, accelerate\n",
            "Successfully installed accelerate-0.29.2 datasets-2.18.0 dill-0.3.8 multiprocess-0.70.16 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.19.3 nvidia-nvjitlink-cu12-12.4.127 nvidia-nvtx-cu12-12.1.105 xxhash-3.4.1\n"
          ]
        }
      ],
      "source": [
        "!pip install datasets accelerate"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "22sQ_oMgEHR9"
      },
      "outputs": [],
      "source": [
        "%%bash\n",
        "!(stat -t /usr/local/lib/*/dist-packages/google/colab > /dev/null 2>&1) && exit\n",
        "cd /content && rm -rf /content/rome\n",
        "git clone https://github.com/kmeng01/rome rome > install.log 2>&1\n",
        "pip install -r /content/rome/scripts/colab_reqs/rome.txt >> install.log 2>&1\n",
        "pip install --upgrade google-cloud-storage >> install.log 2>&1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a3QSWiHz-LZv",
        "outputId": "dad122d0-4825-4cb3-ad95-66be05f5e9e0"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2024-04-16 16:06:05--  https://rome.baulab.info/data/dsets/counterfact.json\n",
            "Resolving rome.baulab.info (rome.baulab.info)... 35.232.255.106\n",
            "Connecting to rome.baulab.info (rome.baulab.info)|35.232.255.106|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 45108470 (43M) [application/json]\n",
            "Saving to: ‘counterfact.json’\n",
            "\n",
            "counterfact.json    100%[===================>]  43.02M   134MB/s    in 0.3s    \n",
            "\n",
            "2024-04-16 16:06:05 (134 MB/s) - ‘counterfact.json’ saved [45108470/45108470]\n",
            "\n"
          ]
        }
      ],
      "source": [
        "!wget 'https://rome.baulab.info/data/dsets/counterfact.json' -O counterfact.json"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "39jdfGmjU9Lo",
        "outputId": "67213407-0a8e-4220-a41c-c9e0f9a5d4ba"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting sentence_transformers\n",
            "  Downloading sentence_transformers-2.6.1-py3-none-any.whl (163 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m163.3/163.3 kB\u001b[0m \u001b[31m941.2 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: transformers<5.0.0,>=4.32.0 in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (4.38.2)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (4.66.2)\n",
            "Requirement already satisfied: torch>=1.11.0 in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (2.2.1+cu121)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (1.25.2)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (1.2.2)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (1.11.4)\n",
            "Requirement already satisfied: huggingface-hub>=0.15.1 in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (0.20.3)\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (9.4.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence_transformers) (3.13.4)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence_transformers) (2023.6.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence_transformers) (2.31.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence_transformers) (6.0.1)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence_transformers) (4.11.0)\n",
            "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence_transformers) (24.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (3.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (3.1.3)\n",
            "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (8.9.2.26)\n",
            "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (12.1.3.1)\n",
            "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (11.0.2.54)\n",
            "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (10.3.2.106)\n",
            "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (11.4.5.107)\n",
            "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (12.1.0.106)\n",
            "Requirement already satisfied: nvidia-nccl-cu12==2.19.3 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (2.19.3)\n",
            "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (12.1.105)\n",
            "Requirement already satisfied: triton==2.2.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (2.2.0)\n",
            "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.10/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.11.0->sentence_transformers) (12.4.127)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.32.0->sentence_transformers) (2023.12.25)\n",
            "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.32.0->sentence_transformers) (0.15.2)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.32.0->sentence_transformers) (0.4.2)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->sentence_transformers) (1.4.0)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->sentence_transformers) (3.4.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.11.0->sentence_transformers) (2.1.5)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.15.1->sentence_transformers) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.15.1->sentence_transformers) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.15.1->sentence_transformers) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.15.1->sentence_transformers) (2024.2.2)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.11.0->sentence_transformers) (1.3.0)\n",
            "Installing collected packages: sentence_transformers\n",
            "Successfully installed sentence_transformers-2.6.1\n"
          ]
        }
      ],
      "source": [
        "!pip install sentence_transformers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PE5xt80KVJf4"
      },
      "outputs": [],
      "source": [
        "IS_COLAB = False\n",
        "ALL_DEPS = False\n",
        "try:\n",
        "    import google.colab, torch, os\n",
        "\n",
        "    IS_COLAB = True\n",
        "    os.chdir(\"/content/rome\")\n",
        "    if not torch.cuda.is_available():\n",
        "        raise Exception(\"Change runtime type to include a GPU.\")\n",
        "except ModuleNotFoundError as _:\n",
        "    pass"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "x4o712m6VL9y"
      },
      "outputs": [],
      "source": [
        "%load_ext autoreload\n",
        "%autoreload 2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KINgP9of80mM"
      },
      "outputs": [],
      "source": [
        "from sentence_transformers import SentenceTransformer\n",
        "import pickle\n",
        "import json\n",
        "import torch\n",
        "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
        "\n",
        "from util import nethook\n",
        "from util.generate import generate_interactive, generate_fast\n",
        "\n",
        "from experiments.py.demo import demo_model_editing, stop_execution"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7JJOCxzA9npu"
      },
      "outputs": [],
      "source": [
        "data_path = '/content/counterfact.json'\n",
        "with open(data_path, 'r') as f:\n",
        "    lines = json.load(f)\n",
        "\n",
        "for i, line in enumerate(lines):\n",
        "    subject = line['requested_rewrite']['subject']\n",
        "    prompts = line['paraphrase_prompts']\n",
        "    new_prompts = []\n",
        "    for prompt in prompts:\n",
        "        prefix = prompt[:prompt.find(subject)]\n",
        "        while '.' in prefix:\n",
        "            prefix = prefix[prefix.find('.')+1:]\n",
        "        while '\\n' in prefix:\n",
        "            prefix = prefix[prefix.find('\\n')+1:]\n",
        "        new_prompt = (prefix+prompt[prompt.find(subject):]).strip()\n",
        "        if 'Category' in new_prompt:\n",
        "            new_prompt = new_prompt[new_prompt.find(subject):]\n",
        "        new_prompts.append(new_prompt)\n",
        "        #print(new_prompt)\n",
        "    lines[i]['paraphrase_prompts'] = new_prompts\n",
        "\n",
        "with open(data_path, 'w') as f:\n",
        "    json.dump(lines, f, indent=2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1TQNvPWsA9gL"
      },
      "outputs": [],
      "source": [
        "data={}\n",
        "with open(data_path, \"r\") as file:\n",
        "    data = json.load(file)\n",
        "\n",
        "train_data =  data[:2000]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pFoJe-VTDdMS",
        "outputId": "93cd5e54-9764-4e6c-ffbe-397169b21c18"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[{'case_id': 0,\n",
              "  'pararel_idx': 2796,\n",
              "  'requested_rewrite': {'prompt': 'The mother tongue of {} is',\n",
              "   'relation_id': 'P103',\n",
              "   'target_new': {'str': 'English', 'id': 'Q1860'},\n",
              "   'target_true': {'str': 'French', 'id': 'Q150'},\n",
              "   'subject': 'Danielle Darrieux'},\n",
              "  'paraphrase_prompts': ['Danielle Darrieux, a native',\n",
              "   'Danielle Darrieux spoke the language'],\n",
              "  'neighborhood_prompts': ['The mother tongue of Léon Blum is',\n",
              "   'The native language of Montesquieu is',\n",
              "   'François Bayrou, a native',\n",
              "   'The native language of Raymond Barre is',\n",
              "   'Michel Rocard is a native speaker of',\n",
              "   'Jacques Chaban-Delmas is a native speaker of',\n",
              "   'The native language of François Bayrou is',\n",
              "   'Maurice Genevoix, speaker of',\n",
              "   'The mother tongue of François Bayrou is',\n",
              "   'Melchior de Vogüé, speaker of'],\n",
              "  'attribute_prompts': ['J.\\xa0R.\\xa0R. Tolkien is a native speaker of',\n",
              "   'The mother tongue of Douglas Adams is',\n",
              "   'The mother tongue of Paul McCartney is',\n",
              "   'Elvis Presley is a native speaker of',\n",
              "   'Barack Obama, speaker of',\n",
              "   'Douglas Adams, speaker of',\n",
              "   'Meryl Streep, a native',\n",
              "   'George Orwell spoke the language',\n",
              "   'George Washington, a native',\n",
              "   'Michael Jackson, a native'],\n",
              "  'generation_prompts': [\"Danielle Darrieux's mother tongue is\",\n",
              "   'Where Danielle Darrieux is from, people speak the language of',\n",
              "   \"Danielle Darrieux's mother tongue is\",\n",
              "   'Danielle Darrieux was born in',\n",
              "   \"Danielle Darrieux's mother tongue is\",\n",
              "   \"Danielle Darrieux's mother tongue is\",\n",
              "   'Danielle Darrieux was born in',\n",
              "   'Where Danielle Darrieux is from, people speak the language of',\n",
              "   'Danielle Darrieux was born in',\n",
              "   'Danielle Darrieux was born in']},\n",
              " {'case_id': 1,\n",
              "  'pararel_idx': 19501,\n",
              "  'requested_rewrite': {'prompt': 'The official religion of {} is',\n",
              "   'relation_id': 'P140',\n",
              "   'target_new': {'str': 'Islam', 'id': 'Q432'},\n",
              "   'target_true': {'str': 'Christianity', 'id': 'Q5043'},\n",
              "   'subject': 'Edwin of Northumbria'},\n",
              "  'paraphrase_prompts': ['Edwin of Northumbria follows the religion of',\n",
              "   'Edwin of Northumbria is affiliated with the religion'],\n",
              "  'neighborhood_prompts': ['The official religion of Charles Aznavour is',\n",
              "   'Nicolas Sarkozy is affiliated with the religion',\n",
              "   'Andrew Johnson is affiliated with the religion',\n",
              "   'The official religion of Paul is',\n",
              "   'Ringo Starr is follower of',\n",
              "   'The official religion of Nicolas Sarkozy is',\n",
              "   'The official religion of Andrew Johnson is',\n",
              "   'Orson Welles is affiliated with the religion',\n",
              "   'Lady Gaga is follower of',\n",
              "   'Quentin Tarantino is affiliated with the religion'],\n",
              "  'attribute_prompts': ['Dave Chappelle is affiliated with the religion',\n",
              "   'Mos Def is follower of',\n",
              "   'Hema Malini is affiliated with the religion',\n",
              "   'The official religion of Mos Def is',\n",
              "   'The official religion of Benazir Bhutto is',\n",
              "   'Benazir Bhutto is follower of',\n",
              "   'Rasul Gamzatov follows the religion of',\n",
              "   'saint follows the religion of',\n",
              "   'Averroes follows the religion of',\n",
              "   'The official religion of Rasul Gamzatov is'],\n",
              "  'generation_prompts': [\"Edwin of Northumbria's religious values strongly emphasize\",\n",
              "   'Edwin of Northumbria worships',\n",
              "   'Edwin of Northumbria regularly attends religious events at the local',\n",
              "   'Edwin of Northumbria regularly attends religious events at the local',\n",
              "   'Edwin of Northumbria regularly attends religious events at the local',\n",
              "   'Edwin of Northumbria regularly attends religious events at the local',\n",
              "   'Edwin of Northumbria regularly attends religious events at the local',\n",
              "   'Edwin of Northumbria worships',\n",
              "   'Edwin of Northumbria worships',\n",
              "   'Edwin of Northumbria regularly attends religious events at the local']},\n",
              " {'case_id': 2,\n",
              "  'pararel_idx': 13704,\n",
              "  'requested_rewrite': {'prompt': '{}, the',\n",
              "   'relation_id': 'P1303',\n",
              "   'target_new': {'str': 'piano', 'id': 'Q5994'},\n",
              "   'target_true': {'str': 'guitar', 'id': 'Q6607'},\n",
              "   'subject': 'Toko Yasuda'},\n",
              "  'paraphrase_prompts': ['Toko Yasuda, performing on the',\n",
              "   'Toko Yasuda plays the instrument'],\n",
              "  'neighborhood_prompts': ['Paul McCartney plays the instrument',\n",
              "   'John Lennon, playing the',\n",
              "   'Elvis Presley, the',\n",
              "   'Douglas Adams, playing the',\n",
              "   'John Lennon performs on the',\n",
              "   'Jimi Hendrix, playing the',\n",
              "   'Ringo Starr, playing the',\n",
              "   'Leonard Cohen plays the',\n",
              "   'Bruce Springsteen, playing the',\n",
              "   'John Lennon plays the'],\n",
              "  'attribute_prompts': ['Robert Radecke performs on the',\n",
              "   'Peter Igelhoff, the',\n",
              "   'Grete von Zieritz, performing on the',\n",
              "   'Anton Rubinstein, playing the',\n",
              "   'Hauschka, playing the',\n",
              "   'Joseph Fischhof, the',\n",
              "   'Paul Badura-Skoda plays the',\n",
              "   'Richard Fall performs on the',\n",
              "   'Justus Frantz, the',\n",
              "   'Laci Boldemann, performing on the'],\n",
              "  'generation_prompts': ['Toko Yasuda produces the most amazing music on the',\n",
              "   'Toko Yasuda is known for',\n",
              "   'Toko Yasuda is known for',\n",
              "   'Toko Yasuda is incredible at',\n",
              "   'Toko Yasuda is incredible at',\n",
              "   'Toko Yasuda produces the most amazing music on the',\n",
              "   'Toko Yasuda produces the most amazing music on the',\n",
              "   'Toko Yasuda is known for',\n",
              "   'Toko Yasuda produces the most amazing music on the',\n",
              "   'Toko Yasuda is incredible at']}]"
            ]
          },
          "execution_count": 10,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "train_data[:3]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WVq5tjUDEzyf"
      },
      "outputs": [],
      "source": [
        "MODEL_NAME = 'gpt2-xl'\n",
        "ALG_NAME= 'ROME'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "ecb1b3f80a3749ca9b1375137c21c558",
            "2c9f1b8a3a0c434991abe5e74577f6e7",
            "7f9533b28ffd416998f7c8216cb7eeed",
            "b4169506de3445eb9219703302d073ed",
            "ee67624927d0439c853170fe50f5bbc8",
            "a91fa038e7874904801346047ac202ff",
            "53c7622310124578a9d3102259079b45",
            "fca68c60fa6c482cbe64929484cc589d",
            "c42bce8bf71040e7995ba0dc1c9f7456",
            "e31f15e0bcd6418dac8e249101a8f573",
            "91ab434064184d289db882c24e86af6a",
            "5f5f0ff6c8764fd6a671b9bd06546cff",
            "39a6b3c12cca41b7a53df90e6d882538",
            "a4b7c7a3fd9444d29a635129feff49d0",
            "b829570cdbff4ae483cb737fd1704275",
            "147fd574b00e42f59ba185cd89cf938f",
            "72c18038466846beb5204683e4438950",
            "0e6d52706330472282c511ff150cbb97",
            "e12d7927ba5440208bd031473d5cb780",
            "b955134a9b9f44578e36d2feac47d114",
            "6c1401242cbe44e1ba6e30d99ac2bb7f",
            "c5a932fd6eb24b6cbb09e3c03f07ce89",
            "b5e0b5d99b3e48f58520c07f31474ff5",
            "7d9c5f83c2354d8cb85bc9765a9219d1",
            "0135b84b3d804180b7ae0cd464a12cd6",
            "04fcf99977ca42238d736bb8f9962199",
            "dd798af4c9774ae8a3b5c93f0c466180",
            "1f258cdd901e4d8cbd4328a87ccebeee",
            "d849f04206cb42f08660f6200628aa5a",
            "6ecee932134d4a4ea13ff6ecd1f2bb93",
            "cf95a2dc546f42c99d06ebd750cd2838",
            "84f2a4694183437dae96e414b4ca036a",
            "606433a460e14a0492eb0cf7113f06ff",
            "e84b28f5eeb14cd8a5e4679539c4c0a9",
            "cc4a672362384a65a671b65474522b6f",
            "94f3261b04cb4402bd7f4a6586e6a6c7",
            "eee70d31e0bd4825b58548037c0140ef",
            "c78edfafed6b4baeaaae1562609d2ba1",
            "eb53cd4e45984671b1d5592ff05c7e20",
            "0963509938314d4487f3fd64b25c22aa",
            "3a81748e530c42e3b3cf3ce38769d057",
            "23d4e8b354b040faa92ef87df33f27a6",
            "1124dbbf961848d6a35b602e8d306bf8",
            "4c22806f41b6458a856b2d1b37a1c9a9",
            "ea313873c85e4d149457cff89998bb8c",
            "f52b3313e7574fb2904674f4e40e9cd2",
            "cc4aaf17191d479fa94f3143aa588183",
            "2ce91981bd33400480ce5765ffca84da",
            "1a74bd9840504e18a4a76d969f0ed80c",
            "bde2c018f6b54961b67a3ba8f26fe440",
            "8d42c6b17bf544d8bb20921aa974bbfb",
            "6eb0c026617a4fd9b1812362d4ecf578",
            "8bba8207a5d949e399584c95632717de",
            "ad05004d32914e4dac625f4be8ad6dc1",
            "ec9332ad95f84f008559889e172e3977",
            "dae5711258244feca861e5d4351ee430",
            "2b562aad74d7409589dd03b771e29325",
            "5ed562607ebc49f08ea44370918d4c81",
            "c0c8b47c44d7462586ef8edfe2ed431c",
            "41fe6c0e0a98455b9bf354a728e900af",
            "df340caf93ff461a914b2b32bb3e76a9",
            "397b99c827254c2385204d2d873916c1",
            "b720c3e53c3d43ccba0588babd207568",
            "5d0347f9ce7d4ee8aced1553bc582ba4",
            "cb3c1bd8170a4e3ba0d2da8966d7a41c",
            "62bd4b3359a14cad82e8c46450f39793",
            "f35a76966e0c40cdb772bce08659f353",
            "98c222f87419461a91569159dc8baf56",
            "a9c8171fb99840df87c1a07c5d6acff8",
            "678cd67aa16a4bbda8cb106ee02b36d6",
            "5d5e34679c3743db8a7f74df47cdf250",
            "b12f726b4d054444890c3abb58e14fff",
            "11a1b727dcab44378b81707bc44a142d",
            "52413c2ce2bd4e62adfee648d1c479d5",
            "3818aaad4cc84e73832730f3bec84a63",
            "e522b33296a24ca2a07a58161c47ab41",
            "8514592d7a69453eb8ce2fdfadedd243"
          ]
        },
        "id": "WvNcznvKE-87",
        "outputId": "8be3c650-2caa-4213-d65f-74d74f0a694c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/689 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ecb1b3f80a3749ca9b1375137c21c558"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model.safetensors:   0%|          | 0.00/6.43G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5f5f0ff6c8764fd6a671b9bd06546cff"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "generation_config.json:   0%|          | 0.00/124 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b5e0b5d99b3e48f58520c07f31474ff5"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/26.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e84b28f5eeb14cd8a5e4679539c4c0a9"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "vocab.json:   0%|          | 0.00/1.04M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ea313873c85e4d149457cff89998bb8c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "dae5711258244feca861e5d4351ee430"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer.json:   0%|          | 0.00/1.36M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f35a76966e0c40cdb772bce08659f353"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "GPT2Config {\n",
              "  \"_name_or_path\": \"gpt2-xl\",\n",
              "  \"activation_function\": \"gelu_new\",\n",
              "  \"architectures\": [\n",
              "    \"GPT2LMHeadModel\"\n",
              "  ],\n",
              "  \"attn_pdrop\": 0.1,\n",
              "  \"bos_token_id\": 50256,\n",
              "  \"embd_pdrop\": 0.1,\n",
              "  \"eos_token_id\": 50256,\n",
              "  \"initializer_range\": 0.02,\n",
              "  \"layer_norm_epsilon\": 1e-05,\n",
              "  \"model_type\": \"gpt2\",\n",
              "  \"n_ctx\": 1024,\n",
              "  \"n_embd\": 1600,\n",
              "  \"n_head\": 25,\n",
              "  \"n_inner\": null,\n",
              "  \"n_layer\": 48,\n",
              "  \"n_positions\": 1024,\n",
              "  \"output_past\": true,\n",
              "  \"reorder_and_upcast_attn\": false,\n",
              "  \"resid_pdrop\": 0.1,\n",
              "  \"scale_attn_by_inverse_layer_idx\": false,\n",
              "  \"scale_attn_weights\": true,\n",
              "  \"summary_activation\": null,\n",
              "  \"summary_first_dropout\": 0.1,\n",
              "  \"summary_proj_to_labels\": true,\n",
              "  \"summary_type\": \"cls_index\",\n",
              "  \"summary_use_proj\": true,\n",
              "  \"task_specific_params\": {\n",
              "    \"text-generation\": {\n",
              "      \"do_sample\": true,\n",
              "      \"max_length\": 50\n",
              "    }\n",
              "  },\n",
              "  \"transformers_version\": \"4.38.2\",\n",
              "  \"use_cache\": true,\n",
              "  \"vocab_size\": 50257\n",
              "}"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ],
      "source": [
        "model, tok = (\n",
        "    AutoModelForCausalLM.from_pretrained(MODEL_NAME, low_cpu_mem_usage=IS_COLAB).to(\n",
        "        \"cuda\"\n",
        "    ),\n",
        "    AutoTokenizer.from_pretrained(MODEL_NAME),\n",
        ")\n",
        "tok.pad_token = tok.eos_token\n",
        "model.config"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1Obwg8UuFGal",
        "outputId": "91961901-dbdb-4341-ccdc-d773ba983c96"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "No model weights to restore: name 'orig_weights' is not defined\n"
          ]
        }
      ],
      "source": [
        "try:\n",
        "    with torch.no_grad():\n",
        "        for k, v in orig_weights.items():\n",
        "            nethook.get_parameter(model, k)[...] = v\n",
        "    print(\"Original model restored\")\n",
        "except NameError as e:\n",
        "    print(f\"No model weights to restore: {e}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "udZDzvMVFHZs"
      },
      "outputs": [],
      "source": [
        "from rome import ROMEHyperParams, apply_rome_to_model\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sWoqfL4PI9TG",
        "outputId": "50b9c1e2-fbef-4295-a496-9fd133a06a5f"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "ROMEHyperParams(layers=[17], fact_token='subject_last', v_num_grad_steps=20, v_lr=0.5, v_loss_layer=47, v_weight_decay=0.5, clamp_norm_factor=4, kl_factor=0.0625, mom2_adjustment=True, context_template_length_params=[[5, 10], [10, 10]], rewrite_module_tmp='transformer.h.{}.mlp.c_proj', layer_module_tmp='transformer.h.{}', mlp_module_tmp='transformer.h.{}.mlp', attn_module_tmp='transformer.h.{}.attn', ln_f_module='transformer.ln_f', lm_head_module='transformer.wte', mom2_dataset='wikipedia', mom2_n_samples=100000, mom2_dtype='float32')"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ],
      "source": [
        "def load_alg(alg_name):\n",
        "  if alg_name == \"ROME\":\n",
        "        return ROMEHyperParams, apply_rome_to_model, \"ROME\", \"\"\n",
        "\n",
        "RewritingParamsClass, apply_method, hparams_prefix, hparams_suffix = load_alg(ALG_NAME)\n",
        "params_name= f'hparams/{hparams_prefix}/{MODEL_NAME}.json'\n",
        "hparams = RewritingParamsClass.from_json(params_name)\n",
        "hparams"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7Al-e1DMN-jA"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import scipy\n",
        "import nltk\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from dsets import AttributeSnippets\n",
        "from util.generate import generate_fast"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3ipr4e1mfuQr",
        "outputId": "3eadc6a4-f898-4a01-e8bb-df55b2e8483c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ],
      "source": [
        "import nltk\n",
        "nltk.download('punkt')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wei3AjGKZKUk"
      },
      "outputs": [],
      "source": [
        "from itertools import chain\n",
        "\n",
        "import nltk\n",
        "import numpy as np\n",
        "import scipy\n",
        "import torch\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
        "\n",
        "from dsets import AttributeSnippets\n",
        "from util.generate import generate_fast\n",
        "from util.perplexity import perplexity"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wENrC1TvLrgJ",
        "outputId": "8d51eb52-f166-4a01-a4e7-0d979d80f715"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2024-04-16 16:09:19--  https://raw.githubusercontent.com/princeton-nlp/MQuAKE/main/datasets/MQuAKE-CF-3k.json\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 16055945 (15M) [text/plain]\n",
            "Saving to: ‘MQuAKE-CF-3k.json’\n",
            "\n",
            "MQuAKE-CF-3k.json   100%[===================>]  15.31M  --.-KB/s    in 0.1s    \n",
            "\n",
            "2024-04-16 16:09:19 (137 MB/s) - ‘MQuAKE-CF-3k.json’ saved [16055945/16055945]\n",
            "\n"
          ]
        }
      ],
      "source": [
        "!wget \"https://raw.githubusercontent.com/princeton-nlp/MQuAKE/main/datasets/MQuAKE-CF-3k.json\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JQRqb1LQL-eM"
      },
      "outputs": [],
      "source": [
        "import json\n",
        "with open('/content/rome/MQuAKE-CF-3k.json', 'r') as f:\n",
        "    dataset = json.load(f)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RAMY32cvMM52",
        "outputId": "b3bfaf8f-02e8-46d5-dd47-6018923a5a9d"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'case_id': 1001,\n",
              " 'requested_rewrite': [{'prompt': '{} is associated with the sport of',\n",
              "   'relation_id': 'P641',\n",
              "   'target_new': {'str': 'basketball', 'id': 'Q5372'},\n",
              "   'target_true': {'str': 'association football', 'id': 'Q2736'},\n",
              "   'subject': 'Saipa F.C.',\n",
              "   'question': 'Which sport is Saipa F.C. associated with?'},\n",
              "  {'prompt': '{} was created in the country of',\n",
              "   'relation_id': 'P495',\n",
              "   'target_new': {'str': 'Spain', 'id': 'Q29'},\n",
              "   'target_true': {'str': 'United States of America', 'id': 'Q30'},\n",
              "   'subject': 'basketball',\n",
              "   'question': 'Which country was basketball created in?'},\n",
              "  {'prompt': 'The official language of {} is',\n",
              "   'relation_id': 'P37',\n",
              "   'target_new': {'str': 'Arabic', 'id': 'Q13955'},\n",
              "   'target_true': {'str': 'Spanish', 'id': 'Q1321'},\n",
              "   'subject': 'Spain',\n",
              "   'question': 'What is the official language of Spain?'}],\n",
              " 'questions': ['What is the official language of the country where Saipa F.C. originates the sport?',\n",
              "  'Which language is used as the official language of the country where Saipa F.C. originates the sport?',\n",
              "  \"In what language does the country of origin of Saipa F.C.'s sport have as its official language?\"],\n",
              " 'answer': 'English',\n",
              " 'answer_alias': ['en', 'eng', 'English language'],\n",
              " 'new_answer': 'Arabic',\n",
              " 'new_answer_alias': ['ar', 'Arabian language', 'Arabic language'],\n",
              " 'single_hops': [{'question': 'Which sport is Saipa F.C. associated with?',\n",
              "   'cloze': 'Saipa F.C. is associated with the sport of',\n",
              "   'answer': 'association football',\n",
              "   'answer_alias': ['football',\n",
              "    'association soccer',\n",
              "    'footie',\n",
              "    'footy',\n",
              "    'futbol',\n",
              "    'futebol',\n",
              "    'soccer']},\n",
              "  {'question': 'Which country was association football created in?',\n",
              "   'cloze': 'association football was created in the country of',\n",
              "   'answer': 'England',\n",
              "   'answer_alias': ['ENG']},\n",
              "  {'question': 'What is the official language of England?',\n",
              "   'cloze': 'The official language of England is',\n",
              "   'answer': 'English',\n",
              "   'answer_alias': ['en', 'eng', 'English language']}],\n",
              " 'new_single_hops': [{'question': 'Which sport is Saipa F.C. associated with?',\n",
              "   'cloze': 'Saipa F.C. is associated with the sport of',\n",
              "   'answer': 'basketball',\n",
              "   'answer_alias': ['BB', 'Basketball', 'basket ball', 'b-ball', 'hoops']},\n",
              "  {'question': 'Which country was basketball created in?',\n",
              "   'cloze': 'basketball was created in the country of',\n",
              "   'answer': 'Spain',\n",
              "   'answer_alias': ['ESP', 'ES', 'Kingdom of Spain', '🇪🇸']},\n",
              "  {'question': 'What is the official language of Spain?',\n",
              "   'cloze': 'The official language of Spain is',\n",
              "   'answer': 'Arabic',\n",
              "   'answer_alias': ['ar', 'Arabian language', 'Arabic language']}],\n",
              " 'orig': {'triples': [['Q782983', 'P641', 'Q2736'],\n",
              "   ['Q2736', 'P495', 'Q21'],\n",
              "   ['Q21', 'P37', 'Q1860']],\n",
              "  'triples_labeled': [['Saipa F.C.', 'sport', 'association football'],\n",
              "   ['association football', 'country of origin', 'England'],\n",
              "   ['England', 'official language', 'English']],\n",
              "  'new_triples': [['Q782983', 'P641', 'Q5372'],\n",
              "   ['Q5372', 'P495', 'Q29'],\n",
              "   ['Q29', 'P37', 'Q13955']],\n",
              "  'new_triples_labeled': [['Saipa F.C.', 'sport', 'basketball'],\n",
              "   ['basketball', 'country of origin', 'Spain'],\n",
              "   ['Spain', 'official language', 'Arabic']],\n",
              "  'edit_triples': [['Q782983', 'P641', 'Q5372'],\n",
              "   ['Q5372', 'P495', 'Q29'],\n",
              "   ['Q29', 'P37', 'Q13955']]}}"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ],
      "source": [
        "dataset[1000]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mKJccbYkuWt8",
        "outputId": "53a087d1-4fa2-48f9-b23a-b0ef7db42b22"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[{'prompt': '{} is associated with the sport of', 'subject': 'Saipa F.C.', 'target_new': {'str': 'basketball'}}, {'prompt': 'The type of music that {} plays is', 'subject': 'Betty Carter', 'target_new': {'str': 'instrumental rock'}}]\n",
            "[['What is the official language of the country where Saipa F.C. originates the sport? Answer:', 'Which language is used as the official language of the country where Saipa F.C. originates the sport? Answer:', \"In what language does the country of origin of Saipa F.C.'s sport have as its official language? Answer:\"], ['Who is the head of state of the country from which the music genre associated with Betty Carter originated? Answer:', \"What is the name of the head of state of the country where the music genre of Betty Carter's style originated? Answer:\", 'Who is the political leader of the country of origin of the music genre identified with Betty Carter? Answer:'], ['What is the capital of the country where the South Atlantic League originated? Answer:', 'In which city is the headquarters of the country where the sport of the South Atlantic League was born? Answer:', 'What is the name of the capital city of the country of origin of the South Atlantic League? Answer:'], ['What is the birthplace of the founder of the religion associated with Jean Bodin? Answer:', 'Where did the founder of the religion Jean Bodin was associated with hail from? Answer:', 'In which city was the founder of the religion that Jean Bodin followed born? Answer:'], [\"Who is the head of government in the location where Chevrolet Cobalt's manufacturer was formed? Answer:\", \"What is the name of the person who serves as head of government in the place of origin of Chevrolet Cobalt's manufacturer? Answer:\", 'Who is the current head of government in the location where the manufacturer of Chevrolet Cobalt was established? Answer:'], ['What is the birthplace of the founder of the religion of Louis XV of France? Answer:', 'In which city was the founder of the religion followed by Louis XV of France born? Answer:', 'Where did the person who founded the religion that Louis XV of France followed originate from? Answer:'], ['What continent is the place of death of the author of \"1984\" located in? Answer:', 'In which continent did the author of \"1984\" pass away? Answer:', 'The writer of the novel \"1984\" died in which continent? Answer:'], ['Which country does the sport that Juninho Pernambucano played originate from? Answer:', 'From which country did the sport, in which Juninho Pernambucano was a specialist, originate? Answer:', 'Where does the sport that Juninho Pernambucano specialized in originally come from? Answer:'], ['Which city is the capital of the country where the performer of \"Firework\" holds citizenship? Answer:', 'What is the capital of the country that the performer of \"Firework\" belongs to? Answer:', 'Where is the capital city of the country that the artist who performed \"Firework\" comes from? Answer:'], [\"What is the official language of the country where Basil D'Oliveira's sport originated? Answer:\", \"In which language is the sport of Basil D'Oliveira's country of origin usually played? Answer:\", \"What language is considered the official language of the country where the sport associated with Basil D'Oliveira originated? Answer:\"], ['From which country does the sport associated with Abet Guidaben originate? Answer:', 'What is the country of origin of the sport in which Abet Guidaben specialized? Answer:', 'Which country is known as the place of origin of the sport that Abet Guidaben played? Answer:'], ['Which country does the sport played by Grégory Coupet originally come from? Answer:', 'What country is the origin of the sport Grégory Coupet played? Answer:', \"From what country did the sport of Grégory Coupet's specialty originate? Answer:\"], ['What is the official language of the country where the genre of Mike Stern was originated? Answer:', \"In which language are the musical works from the country where Mike Stern's genre was originated usually written in? Answer:\", 'Which language is recognized as the official language of the country where the genre of Mike Stern first emerged? Answer:'], ['Where did the founder of the religion associated with Louis of Toulouse work? Answer:', \"What is the workplace of the person who founded Louis of Toulouse's religion? Answer:\", 'In which city did the creator of the religion followed by Louis of Toulouse work? Answer:'], ['Who is the head of state of the country where Don Balón originated? Answer:', 'Which political leader is the head of state of the country where the sport associated with Don Balón was created? Answer:', \"What is the name of the head of state of the country of origin of Don Balón's sport? Answer:\"], ['What country does the head coach of Toronto Blue Jays come from? Answer:', 'From which country does the coach of Toronto Blue Jays originally hail? Answer:', 'What is the country of origin of the head coach of Toronto Blue Jays? Answer:'], ['What is the location of the headquarters of the employer of the author of \"The Principles of Psychology\"? Answer:', 'Where is the headquarters of the company that employed the author of \"The Principles of Psychology\" located? Answer:', 'In which city is the headquarters of the author of \"The Principles of Psychology\"\\'s employer situated? Answer:'], ['What is the official language of the country where the genre of Sturgill Simpson originated? Answer:', 'Which language has official status in the country where the musical genre of Sturgill Simpson was born? Answer:', \"In which language is the country of origin of Sturgill Simpson's genre officially listed? Answer:\"], [\"What is the official language of the country where Lestat de Lioncourt's creator holds a citizenship? Answer:\", 'In which country, which is the citizenship of the creator of Lestat de Lioncourt, is the language spoken as the official language? Answer:', 'Which language is used as the official language in the country of citizenship of the creator of Lestat de Lioncourt? Answer:'], ['Who founded the religion that the author of \"Gesta Danorum\" followed? Answer:', 'What is the name of the person who established the religious faith to which the author of \"Gesta Danorum\" belonged? Answer:', 'Which figure started the religion that the writer of \"Gesta Danorum\" practiced? Answer:'], [\"At which location did the founder of Pedro Arrupe's religion work? Answer:\", 'Where was the workplace of the person who founded the religion associated with Pedro Arrupe? Answer:', 'What is the location where the religion, founded by the person associated with Pedro Arrupe, was active? Answer:'], ['Who is the head of government of the country where the musical genre associated with Kazumi Watanabe originated from? Answer:', 'In which country did the musical genre associated with Kazumi Watanabe originate from and who is its head of government? Answer:', 'Which leader governs the country from which the musical genre associated with Kazumi Watanabe originated? Answer:'], ['What is the location of the workplace of the person who founded the religion of Albert of Saxony? Answer:', 'Where did the person who created the religion of Albert of Saxony work? Answer:', 'What is the place of work of the founder of the religion that Albert of Saxony followed? Answer:'], ['In what city was the person who established the religion of Marcellus II born? Answer:', \"What is the birthplace of the founder of Marcellus II's religion? Answer:\", 'Where did the individual who founded the religion followed by Marcellus II come from? Answer:'], ['Who is the head of government in the place where \"I Dream of Jeannie\" was originally broadcasted? Answer:', 'What is the name of the chief magistrate in the location where \"I Dream of Jeannie\" was first broadcasted? Answer:', 'Who is the head of state of the country where \"I Dream of Jeannie\" was created? Answer:'], ['Which country does the sport that Bill Russell played originate from? Answer:', 'What is the country of origin of the sport that Bill Russell specialized in? Answer:', 'From which country did the sport played by Bill Russell come from? Answer:'], [\"Which country does the Tyronn Lue's sport originate from? Answer:\", 'What is the country of origin of the sport played by Tyronn Lue? Answer:', 'From which country did the sport Tyronn Lue specialized in originate? Answer:'], ['What is the country of origin of the genre associated with the spouse of Kim Kardashian? Answer:', 'From which country does the genre of the partner of Kim Kardashian hail? Answer:', 'Which country is the genre of the partner of Kim Kardashian associated with originally from? Answer:'], ['What is the capital of the country where the sport associated with Lincoln City F.C. originated? Answer:', 'Which city serves as the capital of the country where the sport related to Lincoln City F.C. was founded? Answer:', 'In which city is the capital of the country where the sport linked with Lincoln City F.C. originated? Answer:'], ['What is the capital of the country where the sport that propelled the creation of the Cricket World Cup originated from? Answer:', 'Which city serves as the capital of the country that gave rise to the sport in which the Cricket World Cup is played? Answer:', 'In which city was the sport that led to the inception of Cricket World Cup first played, which is in the capital city of the country? Answer:'], [\"In which continent was the birthplace of Roma Downey's spouse? Answer:\", 'Which continent does the partner of Roma Downey hail from? Answer:', \"What is the continent of origin of Roma Downey's spouse? Answer:\"], ['Which country does the sport played by Shaun Marcum belong to? Answer:', 'From which country does the sport played by Shaun Marcum originate? Answer:', 'What is the country of origin for the sport in which Shaun Marcum played? Answer:'], ['Who is the head of government of the country where the sport associated with Mark Lemke originated? Answer:', 'What is the name of the current head of government of the country where the sport played by Mark Lemke was created? Answer:', \"Which person is the head of government of the country that Mark Lemke's sport originates from? Answer:\"], ['Who is the head of state of the country of origin of the Eredivisie sport? Answer:', \"In which country is the head of state of the Eredivisie's country of origin located? Answer:\", 'What is the name of the chief executive of the country where the Eredivisie sport originated? Answer:'], ['In which country was the sport played by Nigel de Jong originated? Answer:', \"Which country is the sport associated with Nigel de Jong's position played on a team from? Answer:\", 'What is the country of origin of the sport played by Nigel de Jong based on his position played on a team? Answer:'], [\"What is the headquarters location of the institution where Jim Hawkins' creator was educated? Answer:\", 'Where is the educational institution located, which the creator of Jim Hawkins attended with its headquarters? Answer:', 'In what city is the institution located where the creator of Jim Hawkins was educated, and which has its headquarters at that location? Answer:'], ['What is the official language of the country where the sport associated with Dave Cowens originated? Answer:', 'In what language are official transactions conducted in the country of origin for the sport Dave Cowens is associated with? Answer:', 'What is the primary language spoken in the country where the sport Dave Cowens is associated with originated? Answer:'], ['Where was the place of work of the founder of the religion with which Camillo Ruini is associated? Answer:', 'What is the location where the founder of the religion Camillo Ruini was linked with worked? Answer:', \"In which city did the founder of Camillo Ruini's religion work? Answer:\"], [\"What is the work location of the establishment founded by the founder of Desmond Connell's religion? Answer:\", \"Where was the founder of Desmond Connell's religion working when he/she created it? Answer:\", \"In which city did the founder of Desmond Connell's religion work when coming up with it? Answer:\"], [\"Which continent does the country of origin of Leigh Genesis F.C.'s sport belong to? Answer:\", 'What continent is the country of origin of the sport played by Leigh Genesis F.C. found in? Answer:', 'In which continent can the country of origin of the sport played by Leigh Genesis F.C. be located? Answer:'], ['At which location did the founder of the religion that Ignatius Kutu Acheampong was associated with work? Answer:', 'Where did the founder of the religion that Ignatius Kutu Acheampong followed work? Answer:', 'What is the work location of the founder of the religion that Ignatius Kutu Acheampong believed in? Answer:'], [\"From which country does the sport of Bert Trautmann's specialization originate? Answer:\", 'Which country is associated with the sport in which Bert Trautmann plays his specialized position? Answer:', 'What is the country of origin for the sport that Bert Trautmann is involved in? Answer:'], ['Who is the current head of state of the country where Tanduay Rhum Masters was created as a sports team? Answer:', 'What is the name of the current leader of the country of origin of Tanduay Rhum Masters sports team? Answer:', 'In which country, whose current head of state is who, was the Tanduay Rhum Masters sports team first established? Answer:'], ['What is the birthplace of the person who founded the religion Ambrose is associated with? Answer:', 'Which city does the founder of the religion Ambrose was associated with hail from? Answer:', 'In what city was the person who created the religion that Ambrose is linked with born? Answer:'], [\"Who is the head of state of the country where Maccabi Tel Aviv F.C.'s sport originated? Answer:\", 'What is the name of the leader of the country where the sport of Maccabi Tel Aviv F.C. was first played? Answer:', 'Who is the head of state of the country where the sport played by Maccabi Tel Aviv F.C. was first established? Answer:'], ['In what language is \"Typee\", a notable work by a certain author, written in? Answer:', 'What language is used for the work that the author of \"Typee\" is known for? Answer:', 'Which language is attributed to the most well-known work by the author who wrote \"Typee\"? Answer:'], ['Who founded the religion associated with the spouse of Elisabeth Farnese? Answer:', 'Which person established the religion that was followed by the partner of Elisabeth Farnese? Answer:', 'What is the name of the founder of the religion which the spouse of Elisabeth Farnese adhered to? Answer:'], ['In which continent is the country of origin of Supercopa de España located? Answer:', 'Which continent encompasses the country where Supercopa de España originates from? Answer:', 'What is the continent of origin of the country hosting the sporting event Supercopa de España? Answer:'], ['Who is the head of state of the country of origin of the West Midlands (Regional) League? Answer:', 'What is the name of the chief of the state where the sport played by the West Midlands (Regional) League originated? Answer:', 'Who is the leader of the country where the sport played by the West Midlands (Regional) League was founded? Answer:'], ['In what place is the headquarters of the educational institution where the chairperson of the Scottish National Party studied? Answer:', 'What is the location of the educational institution attended by the chairperson of the Scottish National Party, which now serves as the base for headquarters of the institution? Answer:', 'Where is the headquarters located for the institution where the chairperson of the Scottish National Party had their education? Answer:']]\n",
            "['Which sport is Saipa F.C. associated with? Answer:', 'What type of music does Betty Carter play? Answer:', 'Which sport is South Atlantic League associated with? Answer:', 'Which religion is Jean Bodin affiliated with? Answer:', 'Which company is Chevrolet Cobalt produced by? Answer:', 'Which religion is Louis XV of France affiliated with? Answer:', 'Who is the author of 1984? Answer:', 'What position does Juninho Pernambucano play? Answer:', 'Who performed Firework? Answer:', \"Which sport is Basil D'Oliveira associated with? Answer:\", 'What position does Abet Guidaben play? Answer:', 'What position does Grégory Coupet play? Answer:', 'What type of music does Mike Stern play? Answer:', 'Which religion is Louis of Toulouse affiliated with? Answer:', 'Which sport is Don Balón associated with? Answer:', 'Who is the head coach of Toronto Blue Jays? Answer:', 'Who is the author of The Principles of Psychology? Answer:', 'What type of music does Sturgill Simpson play? Answer:', 'Who was Lestat de Lioncourt created by? Answer:', 'Who is the author of Gesta Danorum? Answer:', 'Which religion is Pedro Arrupe affiliated with? Answer:', 'What type of music does Kazumi Watanabe play? Answer:', 'Which religion is Albert of Saxony affiliated with? Answer:', 'Which religion is Marcellus II affiliated with? Answer:', 'Who is the original broadcaster of I Dream of Jeannie? Answer:', 'What position does Bill Russell play? Answer:', 'What position does Tyronn Lue play? Answer:', 'Who is Kim Kardashian married to? Answer:', 'Which sport is Lincoln City F.C. associated with? Answer:', 'Which sport is Cricket World Cup associated with? Answer:', 'Who is Roma Downey married to? Answer:', 'What position does Shaun Marcum play? Answer:', 'Which sport is Mark Lemke associated with? Answer:', 'Which sport is Eredivisie associated with? Answer:', 'What position does Nigel de Jong play? Answer:', 'Who was Jim Hawkins created by? Answer:', 'Which sport is Dave Cowens associated with? Answer:', 'Which religion is Camillo Ruini affiliated with? Answer:', 'Which religion is Desmond Connell affiliated with? Answer:', 'Which sport is Leigh Genesis F.C. associated with? Answer:', 'Which religion is Ignatius Kutu Acheampong affiliated with? Answer:', 'What position does Bert Trautmann play? Answer:', 'Which sport is Tanduay Rhum Masters associated with? Answer:', 'Which religion is Ambrose affiliated with? Answer:', 'Which sport is Maccabi Tel Aviv F.C. associated with? Answer:', 'Who is the author of Typee? Answer:', 'Who is Elisabeth Farnese married to? Answer:', 'Which sport is Supercopa de España associated with? Answer:', 'Which sport is West Midlands (Regional) League associated with? Answer:', 'Who is the chairperson of Scottish National Party? Answer:']\n",
            "[['English', 'en', 'eng', 'English language'], ['Donald Trump', '@realDonaldTrump', 'David Dennison', 'DJT', 'Donald J Trump', 'Donald J. Trump', 'Individual 1', 'Mr Trump', 'POTUS 45', 'President Donald J Trump', 'President Donald J. Trump', 'President Donald John Trump', 'President Donald Trump', 'President Trump', 'The Donald', 'John Miller', 'Trump', 'John Barron', 'Donald John Trump'], ['Washington, D.C.', 'Washington', 'The District', 'D.C.', 'DC', 'District of Columbia', 'Washington D.C.', 'Washington, DC', 'US-DC', 'Washington DC', 'Washington, D.C., U.S.', 'Washington, District of Columbia'], ['Bethlehem'], ['Mike Duggan', 'Michael Edward Duggan'], ['Bethlehem'], ['Europe', 'European continent', 'Old Continent'], ['England', 'ENG'], ['Washington, D.C.', 'Washington', 'The District', 'D.C.', 'DC', 'District of Columbia', 'Washington D.C.', 'Washington, DC', 'US-DC', 'Washington DC', 'Washington, D.C., U.S.', 'Washington, District of Columbia'], ['English', 'en', 'eng', 'English language'], ['United States of America', 'the States', 'the U.S.', 'the United States', 'the United States of America', 'the US', 'the USA', 'U. S.', 'U. S. A.', 'U.S.', 'U.S.A.', 'US of A', 'USA', 'America', 'United States', 'US', 'the U.S. of A', 'the U.S. of America', 'the U.S.A.', 'the US of A', 'the US of America', 'U.S. of America', 'US of America'], ['England', 'ENG'], ['American English', 'AmE', 'en_us', 'en_US', 'en-us', 'en-US', 'English (United States)', 'U.S. English', 'United States English', 'US English'], ['Galilee'], ['Elizabeth II', 'Queen Elizabeth', 'The Queen', 'Queen Elizabeth II', 'Lilibet', 'Queen of England', 'HM Queen Elizabeth II', 'E.R. II', 'E2R', 'EIIR', 'Elizabeth Alexandra Mary', 'Elizabeth Alexandra Mary Windsor', 'Elizabeth II of the United Kingdom', 'Elizabeth the second', 'Elizabeth Windsor', 'ER II', 'ERII', 'Her Royal Highness Princess Elizabeth of York', 'Her Royal Highness The Princess Elizabeth of the United Kingdom', 'Her Royal Highness The Princess Elizabeth, Duchess of Edinburgh', 'Princess Elizabeth of York', 'Princess Elizabeth, Duchess of Edinburgh', 'QE2', 'QEII', 'Queen Elizabeth II of the United Kingdom'], ['United States of America', 'the States', 'the U.S.', 'the United States', 'the United States of America', 'the US', 'the USA', 'U. S.', 'U. S. A.', 'U.S.', 'U.S.A.', 'US of A', 'USA', 'America', 'United States', 'US', 'the U.S. of A', 'the U.S. of America', 'the U.S.A.', 'the US of A', 'the US of America', 'U.S. of America', 'US of America'], ['Cambridge', 'Cambridge, Massachusetts', 'Cambridge, MA'], ['American English', 'AmE', 'en_us', 'en_US', 'en-us', 'en-US', 'English (United States)', 'U.S. English', 'United States English', 'US English'], ['American English', 'AmE', 'en_us', 'en_US', 'en-us', 'en-US', 'English (United States)', 'U.S. English', 'United States English', 'US English'], ['Jesus Christ', \"Yeshu'\", 'God the Son', 'Jesus Christ of Nazareth', 'Jmmanuel', 'Saviour of the world', 'Yahshua', 'Yeshua ben-Yosef', 'Christ', 'Jesus of Nazareth', 'King of Kings', 'Josh', 'Joshua', 'The Saviour', 'Son of God', 'The Messiah', 'Jesus Christ', 'Yehoshua', 'Yeshua', 'Lord of Lords', 'Son of Man', 'Christ Jesus'], ['Galilee'], ['Donald Trump', '@realDonaldTrump', 'David Dennison', 'DJT', 'Donald J Trump', 'Donald J. Trump', 'Individual 1', 'Mr Trump', 'POTUS 45', 'President Donald J Trump', 'President Donald J. Trump', 'President Donald John Trump', 'President Donald Trump', 'President Trump', 'The Donald', 'John Miller', 'Trump', 'John Barron', 'Donald John Trump'], ['Galilee'], ['Bethlehem'], ['Bill de Blasio', 'Bill deBlasio', 'Warren Wilhelm Jr.', 'Warren Wilhelm, Jr.'], ['United States of America', 'the States', 'the U.S.', 'the United States', 'the United States of America', 'the US', 'the USA', 'U. S.', 'U. S. A.', 'U.S.', 'U.S.A.', 'US of A', 'USA', 'America', 'United States', 'US', 'the U.S. of A', 'the U.S. of America', 'the U.S.A.', 'the US of A', 'the US of America', 'U.S. of America', 'US of America'], ['United States of America', 'the States', 'the U.S.', 'the United States', 'the United States of America', 'the US', 'the USA', 'U. S.', 'U. S. A.', 'U.S.', 'U.S.A.', 'US of A', 'USA', 'America', 'United States', 'US', 'the U.S. of A', 'the U.S. of America', 'the U.S.A.', 'the US of A', 'the US of America', 'U.S. of America', 'US of America'], ['United States of America', 'the States', 'the U.S.', 'the United States', 'the United States of America', 'the US', 'the USA', 'U. S.', 'U. S. A.', 'U.S.', 'U.S.A.', 'US of A', 'USA', 'America', 'United States', 'US', 'the U.S. of A', 'the U.S. of America', 'the U.S.A.', 'the US of A', 'the US of America', 'U.S. of America', 'US of America'], ['London', 'London, United Kingdom', 'Greater London', 'London, England', 'London U.K.', 'London UK', 'London, UK'], ['London', 'London, United Kingdom', 'Greater London', 'London, England', 'London U.K.', 'London UK', 'London, UK'], ['Europe', 'European continent', 'Old Continent'], ['United States of America', 'the States', 'the U.S.', 'the United States', 'the United States of America', 'the US', 'the USA', 'U. S.', 'U. S. A.', 'U.S.', 'U.S.A.', 'US of A', 'USA', 'America', 'United States', 'US', 'the U.S. of A', 'the U.S. of America', 'the U.S.A.', 'the US of A', 'the US of America', 'U.S. of America', 'US of America'], ['Donald Trump', '@realDonaldTrump', 'David Dennison', 'DJT', 'Donald J Trump', 'Donald J. Trump', 'Individual 1', 'Mr Trump', 'POTUS 45', 'President Donald J Trump', 'President Donald J. Trump', 'President Donald John Trump', 'President Donald Trump', 'President Trump', 'The Donald', 'John Miller', 'Trump', 'John Barron', 'Donald John Trump'], ['Elizabeth II', 'Queen Elizabeth', 'The Queen', 'Queen Elizabeth II', 'Lilibet', 'Queen of England', 'HM Queen Elizabeth II', 'E.R. II', 'E2R', 'EIIR', 'Elizabeth Alexandra Mary', 'Elizabeth Alexandra Mary Windsor', 'Elizabeth II of the United Kingdom', 'Elizabeth the second', 'Elizabeth Windsor', 'ER II', 'ERII', 'Her Royal Highness Princess Elizabeth of York', 'Her Royal Highness The Princess Elizabeth of the United Kingdom', 'Her Royal Highness The Princess Elizabeth, Duchess of Edinburgh', 'Princess Elizabeth of York', 'Princess Elizabeth, Duchess of Edinburgh', 'QE2', 'QEII', 'Queen Elizabeth II of the United Kingdom'], ['England', 'ENG'], ['Edinburgh', 'Edina', 'Edinburg', 'City of Edinburgh', 'Modern Athens', 'Auld Reekie', 'Athens of the North', 'Dùn Èideann', 'Edinburgh, Scotland'], ['American English', 'AmE', 'en_us', 'en_US', 'en-us', 'en-US', 'English (United States)', 'U.S. English', 'United States English', 'US English'], ['Galilee'], ['Galilee'], ['Europe', 'European continent', 'Old Continent'], ['Galilee'], ['England', 'ENG'], ['Donald Trump', '@realDonaldTrump', 'David Dennison', 'DJT', 'Donald J Trump', 'Donald J. Trump', 'Individual 1', 'Mr Trump', 'POTUS 45', 'President Donald J Trump', 'President Donald J. Trump', 'President Donald John Trump', 'President Donald Trump', 'President Trump', 'The Donald', 'John Miller', 'Trump', 'John Barron', 'Donald John Trump'], ['Bethlehem'], ['Elizabeth II', 'Queen Elizabeth', 'The Queen', 'Queen Elizabeth II', 'Lilibet', 'Queen of England', 'HM Queen Elizabeth II', 'E.R. II', 'E2R', 'EIIR', 'Elizabeth Alexandra Mary', 'Elizabeth Alexandra Mary Windsor', 'Elizabeth II of the United Kingdom', 'Elizabeth the second', 'Elizabeth Windsor', 'ER II', 'ERII', 'Her Royal Highness Princess Elizabeth of York', 'Her Royal Highness The Princess Elizabeth of the United Kingdom', 'Her Royal Highness The Princess Elizabeth, Duchess of Edinburgh', 'Princess Elizabeth of York', 'Princess Elizabeth, Duchess of Edinburgh', 'QE2', 'QEII', 'Queen Elizabeth II of the United Kingdom'], ['English', 'en', 'eng', 'English language'], ['Jesus Christ', \"Yeshu'\", 'God the Son', 'Jesus Christ of Nazareth', 'Jmmanuel', 'Saviour of the world', 'Yahshua', 'Yeshua ben-Yosef', 'Christ', 'Jesus of Nazareth', 'King of Kings', 'Josh', 'Joshua', 'The Saviour', 'Son of God', 'The Messiah', 'Jesus Christ', 'Yehoshua', 'Yeshua', 'Lord of Lords', 'Son of Man', 'Christ Jesus'], ['Europe', 'European continent', 'Old Continent'], ['Elizabeth II', 'Queen Elizabeth', 'The Queen', 'Queen Elizabeth II', 'Lilibet', 'Queen of England', 'HM Queen Elizabeth II', 'E.R. II', 'E2R', 'EIIR', 'Elizabeth Alexandra Mary', 'Elizabeth Alexandra Mary Windsor', 'Elizabeth II of the United Kingdom', 'Elizabeth the second', 'Elizabeth Windsor', 'ER II', 'ERII', 'Her Royal Highness Princess Elizabeth of York', 'Her Royal Highness The Princess Elizabeth of the United Kingdom', 'Her Royal Highness The Princess Elizabeth, Duchess of Edinburgh', 'Princess Elizabeth of York', 'Princess Elizabeth, Duchess of Edinburgh', 'QE2', 'QEII', 'Queen Elizabeth II of the United Kingdom'], ['Glasgow', 'Glasgow, Scotland', 'Glasgow, UK']]\n",
            "[['Spain', 'ESP', 'ES', 'Kingdom of Spain', '🇪🇸', 'association football', 'United States of America', 'Spanish'], ['United States of America', 'the States', 'the U.S.', 'the United States', 'the United States of America', 'the US', 'the USA', 'U. S.', 'U. S. A.', 'U.S.', 'U.S.A.', 'US of A', 'USA', 'America', 'United States', 'US', 'the U.S. of A', 'the U.S. of America', 'the U.S.A.', 'the US of A', 'the US of America', 'U.S. of America', 'US of America', 'jazz', 'Donald Trump'], ['Spain', 'ESP', 'ES', 'Kingdom of Spain', '🇪🇸', 'baseball', 'United States of America', 'Madrid'], ['Mao Zedong', 'Mao Runzhi', 'Chairman Mao Zedong', 'Lun-chi', 'Mao Jun-chih', 'Máo Rùnzhī', 'Mao Zédong', 'Mau Zerdong', 'Mouh Jaahk-dung', 'Chairman Mao', 'Jeon-zi', 'Jun-chih', 'Mao', 'Mao Tse-tung', 'Mao Ze Dong', 'Máo Zédōng', 'Máo Zhǔxí', 'Mao² Tsê²-tung¹', 'Maozedong', 'Mau Tzerdong', 'Mau Zeh-ton', 'Máu Zéh-ton', 'Mo Chhet-tung', 'Mô Chhe̍t-tûng', 'Mo Tek-tong', 'Mô͘ Te̍k-tong', 'Moo Tik-tang', 'Môo Ti̍k-tang', 'Mou Zaak-dung', 'Rùnzhī', 'Tse Toung Mao', 'ㄇㄠˊ ㄗㄜˊ ㄉㄨㄥ', '毛主席', '毛泽东', '毛澤東', '润之', '潤之', 'Catholic Church', 'Gautama Buddha'], ['Moscow', 'Moskva', 'Moscow, Russia', 'City of Moscow', 'Moscow, Russian Federation', 'Moscow, Russian SFSR', 'Moscow, Soviet Union', 'Moscow, USSR', 'Moskva Federal City, Russia', 'Moskva, Rossiya', 'Moskva, Russia', 'Москва', 'Moskovĭ', 'Moskve', 'Moskvë', 'Moskvi', 'Mosky'], ['John Wesley'], ['Bucharest', 'Little Paris', 'București', 'Bucharest, Romania', 'Bucuresti', 'Bucureşti', 'Paris of the East', 'Буцуреșти', 'London', 'Europe'], ['Gaelic football', 'association football', 'Ireland'], ['United Kingdom', 'Britain', 'UK', 'G. B.', 'GBR', 'Great Britain', 'GB', 'G.B.', 'United Kingdom of Great Britain and Northern Ireland', 'G B', 'G B R', 'G.B.R.', '🇬🇧', 'Great Britain and Northern Ireland', 'The UK', 'The United Kingdom of Great Britain and Northern Ireland', 'U K', 'U. K.', 'U.K.', 'United States of America', 'London'], ['Spain', 'ESP', 'ES', 'Kingdom of Spain', '🇪🇸', 'cricket', 'United States of America', 'Spanish'], ['baseball', '⚾', \"America's pastime\", 'center', 'United States of America'], ['Muay Thai', 'Muaythai', 'Thai boxing', 'goalkeeper', 'ice hockey', 'Thailand'], ['United Kingdom', 'Britain', 'UK', 'G. B.', 'GBR', 'Great Britain', 'GB', 'G.B.', 'United Kingdom of Great Britain and Northern Ireland', 'G B', 'G B R', 'G.B.R.', '🇬🇧', 'Great Britain and Northern Ireland', 'The UK', 'The United Kingdom of Great Britain and Northern Ireland', 'U K', 'U. K.', 'U.K.', 'jazz', 'United States of America', 'English'], ['Confucius', 'Kong Fuzi', 'Kong Zhongni', 'Kongfuzi', 'Kongqiu', 'Kung Chung-ni', 'Kung Fu Tzu', 'Kung Fu-tse', 'Kungfutse', 'Zhongni', 'Kongzi', 'Konfuzius', 'Kong Qiu', 'Cheu-kong', 'Chung-ni', 'Confucio', 'Cong-tse', \"K'ung Chung-ni\", \"K'ung Fu-tse\", \"K'ung-fu-tzu\", \"K'ung-tzu\", 'Kong Fu Zi'], ['United Kingdom', 'Britain', 'UK', 'G. B.', 'GBR', 'Great Britain', 'GB', 'G.B.', 'United Kingdom of Great Britain and Northern Ireland', 'G B', 'G B R', 'G.B.R.', '🇬🇧', 'Great Britain and Northern Ireland', 'The UK', 'The United Kingdom of Great Britain and Northern Ireland', 'U K', 'U. K.', 'U.K.', 'association football', 'United States of America', 'Elizabeth II'], ['association football', 'football', 'association soccer', 'footie', 'footy', 'futbol', 'futebol', 'soccer', 'John Gibbons', 'England'], ['Johns Hopkins University', 'Johns Hopkins', 'Hopkins', 'JHU', 'jhu.edu', 'The Johns Hopkins University', 'Harvard University', 'Baltimore'], ['Jamaica', 'JAM', 'JA', '🇯🇲', 'Commonwealth of Jamaica', 'jm'], ['Great Britain', 'UK', 'GB', 'Kingdom of Great Britain', 'UKGB', 'United Kingdom of Great Britain'], ['Catholic Church', 'Roman Catholic Church', 'Roman Apostolic Catholic Church', 'Saxo Grammaticus', 'Hinduism', 'Jesus Christ'], ['Romain Rolland', 'Catholic Church', 'John Wesley'], ['Jamaica', 'JAM', 'JA', '🇯🇲', 'Commonwealth of Jamaica', 'jm'], ['Romain Rolland', 'Catholic Church', 'John Wesley'], ['John Wesley'], ['Los Angeles', 'L.A.', 'City of Angels', 'Los Angeles, California', 'La La Land', 'LA', 'Los Ángeles', 'Pink City', 'City of Los Angeles', 'Tinsel Town', 'Los Angeles, CA', 'Double Dubuque', 'LA, California', 'The town of Our Lady the Queen of the Angels of the Little Portion', 'NBC', 'New York City'], ['cricket', 'Cricket Sport', 'basketball', 'England'], ['cricket', 'Cricket Sport', 'point guard', 'rugby union', 'England'], ['hip hop music', 'hip hop', 'hip-hop music', 'hip-hop'], ['United Kingdom', 'Britain', 'UK', 'G. B.', 'GBR', 'Great Britain', 'GB', 'G.B.', 'United Kingdom of Great Britain and Northern Ireland', 'G B', 'G B R', 'G.B.R.', '🇬🇧', 'Great Britain and Northern Ireland', 'The UK', 'The United Kingdom of Great Britain and Northern Ireland', 'U K', 'U. K.', 'U.K.', 'association football', 'United States of America', 'London'], ['Netherlands', 'NL', 'Holland', 'Nederland', 'NED', 'nl', 'the Netherlands', '🇳🇱', 'Netherlands (after 1945)', 'England', 'Amsterdam'], ['Hong Kong', 'HKG', 'HK', 'Hongkong', 'hk', '🇭🇰', 'China Hong Kong', 'HKSAR', 'Hong Kong SAR', 'Hong Kong SAR, China', 'Hong Kong Special Administrative Region', 'Hong Kong, China', 'xiang gang', 'Mark Burnett', 'New York City', 'Asia'], ['sumo', 'starting pitcher', 'American football', 'Japan'], ['United Kingdom', 'Britain', 'UK', 'G. B.', 'GBR', 'Great Britain', 'GB', 'G.B.', 'United Kingdom of Great Britain and Northern Ireland', 'G B', 'G B R', 'G.B.R.', '🇬🇧', 'Great Britain and Northern Ireland', 'The UK', 'The United Kingdom of Great Britain and Northern Ireland', 'U K', 'U. K.', 'U.K.'], ['United Kingdom', 'Britain', 'UK', 'G. B.', 'GBR', 'Great Britain', 'GB', 'G.B.', 'United Kingdom of Great Britain and Northern Ireland', 'G B', 'G B R', 'G.B.R.', '🇬🇧', 'Great Britain and Northern Ireland', 'The UK', 'The United Kingdom of Great Britain and Northern Ireland', 'U K', 'U. K.', 'U.K.', 'association football', 'United States of America', 'Elizabeth II'], ['rugby union', 'rugby', 'midfielder', 'baseball', 'England'], ['University of Edinburgh', 'The University of Edinburgh', 'Edinburgh University', 'ed.ac.uk'], ['Spain', 'ESP', 'ES', 'Kingdom of Spain', '🇪🇸', 'United States of America', 'Spanish'], ['Mario Monti', 'Catholic Church', 'L. Ron Hubbard'], ['John Knox'], ['Hong Kong', 'HKG', 'HK', 'Hongkong', 'hk', '🇭🇰', 'China Hong Kong', 'HKSAR', 'Hong Kong SAR', 'Hong Kong SAR, China', 'Hong Kong Special Administrative Region', 'Hong Kong, China', 'xiang gang', 'England', 'Asia'], ['Adolfo Suárez', 'Adolfo Suarez', 'Adolfo Suarez Gonzalez'], ['Muay Thai', 'Muaythai', 'Thai boxing', 'goalkeeper', 'ice hockey', 'Thailand'], ['United Kingdom', 'Britain', 'UK', 'G. B.', 'GBR', 'Great Britain', 'GB', 'G.B.', 'United Kingdom of Great Britain and Northern Ireland', 'G B', 'G B R', 'G.B.R.', '🇬🇧', 'Great Britain and Northern Ireland', 'The UK', 'The United Kingdom of Great Britain and Northern Ireland', 'U K', 'U. K.', 'U.K.', 'basketball', 'United States of America', 'Elizabeth II'], ['John Wesley'], ['United Kingdom', 'Britain', 'UK', 'G. B.', 'GBR', 'Great Britain', 'GB', 'G.B.', 'United Kingdom of Great Britain and Northern Ireland', 'G B', 'G B R', 'G.B.R.', '🇬🇧', 'Great Britain and Northern Ireland', 'The UK', 'The United Kingdom of Great Britain and Northern Ireland', 'U K', 'U. K.', 'U.K.', 'association football', 'United States of America', 'Elizabeth II'], ['Moby-Dick', 'The Whale', 'Moby Dick', 'Moby-Dick; or, The Whale'], ['Catholic Church', 'Roman Catholic Church', 'Roman Apostolic Catholic Church'], ['Hong Kong', 'HKG', 'HK', 'Hongkong', 'hk', '🇭🇰', 'China Hong Kong', 'HKSAR', 'Hong Kong SAR', 'Hong Kong SAR, China', 'Hong Kong Special Administrative Region', 'Hong Kong, China', 'xiang gang', 'England', 'Asia'], ['United Kingdom', 'Britain', 'UK', 'G. B.', 'GBR', 'Great Britain', 'GB', 'G.B.', 'United Kingdom of Great Britain and Northern Ireland', 'G B', 'G B R', 'G.B.R.', '🇬🇧', 'Great Britain and Northern Ireland', 'The UK', 'The United Kingdom of Great Britain and Northern Ireland', 'U K', 'U. K.', 'U.K.', 'association football', 'United States of America', 'Elizabeth II'], ['Vilnius University', 'Universitas Vilnensis', 'Vilniaus universitetas', 'Vilniaus Universitetas', 'vu.lt', 'Nicola Sturgeon', 'Vilnius']]\n"
          ]
        }
      ],
      "source": [
        "request =[]\n",
        "multi_hop_prompts =[]\n",
        "single_hop_prompts =[]\n",
        "multi_hop_answers=[]\n",
        "post_multi_ans =[]\n",
        "for cases in dataset[1000:1050]:\n",
        "\n",
        "  temp = cases['requested_rewrite'][0]\n",
        "\n",
        "  temp1={}\n",
        "  temp1['prompt'] = temp['prompt']\n",
        "  temp1['subject'] = temp['subject']\n",
        "  temp1['target_new']={}\n",
        "  temp1['target_new']['str'] =temp['target_new']['str']\n",
        "  request.append(temp1)\n",
        "  temp3 =[]\n",
        "  for prompt in cases['questions']:\n",
        "    prompt += ' Answer:'\n",
        "    temp3.append(prompt)\n",
        "  multi_hop_prompts.append(temp3)\n",
        "  single_hop_prompts.append(cases['new_single_hops'][0]['question'] + \" Answer:\")\n",
        "  t1= [cases['answer']]\n",
        "  t2=cases['answer_alias']\n",
        "  t=[]\n",
        "  if len(cases['requested_rewrite'])>1:\n",
        "    for i in range(len(cases['requested_rewrite'])):\n",
        "      t.append(cases['requested_rewrite'][i]['target_true']['str'])\n",
        "  post_multi_ans.append([cases[\"new_single_hops\"][1][\"answer\"]] + cases[\"new_single_hops\"][1][\"answer_alias\"]+t)\n",
        "  multi_hop_answers.append(t1+t2)\n",
        "\n",
        "\n",
        "print(request[:2])\n",
        "print(multi_hop_prompts)\n",
        "print(single_hop_prompts)\n",
        "print(multi_hop_answers)\n",
        "print(post_multi_ans)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K9yBbBeXvmgr",
        "outputId": "13df5ac9-104c-488d-96d8-2c4f65445a5a"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['What is the official language of the country where Saipa F.C. originates the sport? Answer: English. Why do you',\n",
              "  'Which language is used as the official language of the country where Saipa F.C. originates the sport? Answer: Spanish. ',\n",
              "  \"In what language does the country of origin of Saipa F.C.'s sport have as its official language? Answer: English. \\n\"],\n",
              " ['Who is the head of state of the country from which the music genre associated with Betty Carter originated? Answer: The head of state of the country of',\n",
              "  \"What is the name of the head of state of the country where the music genre of Betty Carter's style originated? Answer: The Republic of the Marshall\",\n",
              "  'Who is the political leader of the country of origin of the music genre identified with Betty Carter? Answer: The leader of the country of origin of the'],\n",
              " ['What is the capital of the country where the South Atlantic League originated? Answer: The capital was New York City (the original capital was Philadelphia) Answer',\n",
              "  'In which city is the headquarters of the country where the sport of the South Atlantic League was born? Answer: New Bedford, Massachusetts. The New Bedford',\n",
              "  'What is the name of the capital city of the country of origin of the South Atlantic League? Answer: Cape Town What is the name of'],\n",
              " ['What is the birthplace of the founder of the religion associated with Jean Bodin? Answer: The birthplace of the founder of the religion associated with Jean Bod',\n",
              "  'Where did the founder of the religion Jean Bodin was associated with hail from? Answer: Jean Bodin was a French Catholic priest who died',\n",
              "  'In which city was the founder of the religion that Jean Bodin followed born? Answer: Paris In which city was the founder of the religion'],\n",
              " [\"Who is the head of government in the location where Chevrolet Cobalt's manufacturer was formed? Answer: The head of government is a person who is responsible\",\n",
              "  \"What is the name of the person who serves as head of government in the place of origin of Chevrolet Cobalt's manufacturer? Answer: The person who\",\n",
              "  'Who is the current head of government in the location where the manufacturer of Chevrolet Cobalt was established? Answer: The current head of government in the location'],\n",
              " ['What is the birthplace of the founder of the religion of Louis XV of France? Answer: Louis XV of France was born at the village of La Ro',\n",
              "  'In which city was the founder of the religion followed by Louis XV of France born? Answer: In the village of Saint-Denis-de-',\n",
              "  \"Where did the person who founded the religion that Louis XV of France followed originate from? Answer: from the Jews. It's a fact that Louis,\"],\n",
              " ['What continent is the place of death of the author of \"1984\" located in? Answer: The continent which is called Europe. What is',\n",
              "  'In which continent did the author of \"1984\" pass away? Answer: \"The author of \\'1984\\' passed away on the 20th',\n",
              "  'The writer of the novel \"1984\" died in which continent? Answer: in London, England. The book was published in Britain in 1953 and in'],\n",
              " ['Which country does the sport that Juninho Pernambucano played originate from? Answer: Brazil Who is the only player in the Brazilian',\n",
              "  'From which country did the sport, in which Juninho Pernambucano was a specialist, originate? Answer: It came to Brazil from Portugal',\n",
              "  'Where does the sport that Juninho Pernambucano specialized in originally come from? Answer: The first recorded mention of it is in a poem'],\n",
              " ['Which city is the capital of the country where the performer of \"Firework\" holds citizenship? Answer: The capital of the country is the place where',\n",
              "  'What is the capital of the country that the performer of \"Firework\" belongs to? Answer: The country that the performer of \"Firework\"',\n",
              "  'Where is the capital city of the country that the artist who performed \"Firework\" comes from? Answer: The city of Los Angeles, CA.'],\n",
              " [\"What is the official language of the country where Basil D'Oliveira's sport originated? Answer: Portuguese. What is the official language of the\",\n",
              "  \"In which language is the sport of Basil D'Oliveira's country of origin usually played? Answer: Spanish Which of the following countries\",\n",
              "  \"What language is considered the official language of the country where the sport associated with Basil D'Oliveira originated? Answer: Brazil. Which\"],\n",
              " ['From which country does the sport associated with Abet Guidaben originate? Answer: The sport originated from the Netherlands. Abet Guidab',\n",
              "  'What is the country of origin of the sport in which Abet Guidaben specialized? Answer: It is not clear if he is a citizen of',\n",
              "  'Which country is known as the place of origin of the sport that Abet Guidaben played? Answer: France. Abet Guidab'],\n",
              " ['Which country does the sport played by Grégory Coupet originally come from? Answer: France How many players in the French team are',\n",
              "  'What country is the origin of the sport Grégory Coupet played? Answer: The sport is known to have been invented by the French,',\n",
              "  \"From what country did the sport of Grégory Coupet's specialty originate? Answer: The Netherlands. In 1848, Coupet was one\"],\n",
              " ['What is the official language of the country where the genre of Mike Stern was originated? Answer: French, but the first known publication in French is the',\n",
              "  \"In which language are the musical works from the country where Mike Stern's genre was originated usually written in? Answer: in English. How many\",\n",
              "  'Which language is recognized as the official language of the country where the genre of Mike Stern first emerged? Answer: English (official). How many'],\n",
              " ['Where did the founder of the religion associated with Louis of Toulouse work? Answer: In the year 1170, in a monastery of St.',\n",
              "  \"What is the workplace of the person who founded Louis of Toulouse's religion? Answer: the workplace of a person who has a religious belief that\",\n",
              "  'In which city did the creator of the religion followed by Louis of Toulouse work? Answer: In the city of Tours. \\nWhat did'],\n",
              " ['Who is the head of state of the country where Don Balón originated? Answer: The head of state of Colombia is the president. The country where',\n",
              "  'Which political leader is the head of state of the country where the sport associated with Don Balón was created? Answer: The first president of Colombia.',\n",
              "  \"What is the name of the head of state of the country of origin of Don Balón's sport? Answer: Spain. What country is\"],\n",
              " ['What country does the head coach of Toronto Blue Jays come from? Answer: The U.S. What country does the Blue Jays manager come',\n",
              "  'From which country does the coach of Toronto Blue Jays originally hail? Answer: India. The team has been playing in the International Cricket Council (ICC',\n",
              "  'What is the country of origin of the head coach of Toronto Blue Jays? Answer: Toronto, Canada Answer: Toronto, Canada What was the'],\n",
              " ['What is the location of the headquarters of the employer of the author of \"The Principles of Psychology\"? Answer: The author\\'s office. The author\\'s',\n",
              "  'Where is the headquarters of the company that employed the author of \"The Principles of Psychology\" located? Answer: The headquarters of the company that employed the',\n",
              "  'In which city is the headquarters of the author of \"The Principles of Psychology\"\\'s employer situated? Answer: London. Answer: London. '],\n",
              " ['What is the official language of the country where the genre of Sturgill Simpson originated? Answer: English.A man has been arrested for threatening',\n",
              "  'Which language has official status in the country where the musical genre of Sturgill Simpson was born? Answer: In the United States, \"',\n",
              "  \"In which language is the country of origin of Sturgill Simpson's genre officially listed? Answer: In both the U.S. and UK,\"],\n",
              " [\"What is the official language of the country where Lestat de Lioncourt's creator holds a citizenship? Answer:\",\n",
              "  'In which country, which is the citizenship of the creator of Lestat de Lioncourt, is the language spoken as the official language? Answer:',\n",
              "  'Which language is used as the official language in the country of citizenship of the creator of Lestat de Lioncourt? Answer:'],\n",
              " ['Who founded the religion that the author of \"Gesta Danorum\" followed? Answer: \"The Jews.\" The author of \"Gesta',\n",
              "  'What is the name of the person who established the religious faith to which the author of \"Gesta Danorum\" belonged? Answer: The author himself',\n",
              "  'Which figure started the religion that the writer of \"Gesta Danorum\" practiced? Answer: The first Christian martyr. \"Gesta\"'],\n",
              " [\"At which location did the founder of Pedro Arrupe's religion work? Answer: In the middle of nowhere, on a mountain called El Castor\",\n",
              "  'Where was the workplace of the person who founded the religion associated with Pedro Arrupe? Answer: The workplace of the founder is in the United States',\n",
              "  'What is the location where the religion, founded by the person associated with Pedro Arrupe, was active? Answer: The location of the religion was'],\n",
              " ['Who is the head of government of the country where the musical genre associated with Kazumi Watanabe originated from? Answer: This is actually',\n",
              "  'In which country did the musical genre associated with Kazumi Watanabe originate from and who is its head of government? Answer: Kazumi Watan',\n",
              "  'Which leader governs the country from which the musical genre associated with Kazumi Watanabe originated? Answer: It is not a question of whether the musical'],\n",
              " ['What is the location of the workplace of the person who founded the religion of Albert of Saxony? Answer: The location of the founder of the religion',\n",
              "  'Where did the person who created the religion of Albert of Saxony work? Answer: He was a Jew. The Jews are a people who',\n",
              "  'What is the place of work of the founder of the religion that Albert of Saxony followed? Answer: In the same place as that of the founder'],\n",
              " ['In what city was the person who established the religion of Marcellus II born? Answer: Rome. Answer: Rome. What was the',\n",
              "  \"What is the birthplace of the founder of Marcellus II's religion? Answer: Rome. Answer: The Romans, who were the first\",\n",
              "  'Where did the individual who founded the religion followed by Marcellus II come from? Answer: From Rome. The Roman Empire The'],\n",
              " ['Who is the head of government in the place where \"I Dream of Jeannie\" was originally broadcasted? Answer: the President of the United States',\n",
              "  'What is the name of the chief magistrate in the location where \"I Dream of Jeannie\" was first broadcasted? Answer: It\\'s \"Chief',\n",
              "  'Who is the head of state of the country where \"I Dream of Jeannie\" was created? Answer: The head of state of the country where'],\n",
              " ['Which country does the sport that Bill Russell played originate from? Answer: The USA, and the NBA is the biggest league of all. In this case',\n",
              "  'What is the country of origin of the sport that Bill Russell specialized in? Answer: China. The Chinese have been playing basketball in America for centuries.',\n",
              "  'From which country did the sport played by Bill Russell come from? Answer: Russia. How did he get his nickname \"the Wizard\"? Answer'],\n",
              " [\"Which country does the Tyronn Lue's sport originate from? Answer: The Tyronn Lue's sport originated from South Sudan\",\n",
              "  'What is the country of origin of the sport played by Tyronn Lue? Answer: United Kingdom Which team is known for playing the',\n",
              "  \"From which country did the sport Tyronn Lue specialized in originate? Answer: It's a question that has been asked of many NBA players over\"],\n",
              " ['What is the country of origin of the genre associated with the spouse of Kim Kardashian? Answer: South Korea. According to the Korean Central News',\n",
              "  'From which country does the genre of the partner of Kim Kardashian hail? Answer: South Korea. The Korean media is filled with photos of the \"Miss',\n",
              "  'Which country is the genre of the partner of Kim Kardashian associated with originally from? Answer: South Korea. The first Kim Kardashian and Kanye West'],\n",
              " ['What is the capital of the country where the sport associated with Lincoln City F.C. originated? Answer: Lincoln City, Missouri Answer: Lincoln City',\n",
              "  'Which city serves as the capital of the country where the sport related to Lincoln City F.C. was founded? Answer: Lincoln City, Nebraska\\n',\n",
              "  'In which city is the capital of the country where the sport linked with Lincoln City F.C. originated? Answer: Lincoln, Illinois. '],\n",
              " ['What is the capital of the country where the sport that propelled the creation of the Cricket World Cup originated from? Answer:',\n",
              "  'Which city serves as the capital of the country that gave rise to the sport in which the Cricket World Cup is played? Answer:',\n",
              "  'In which city was the sport that led to the inception of Cricket World Cup first played, which is in the capital city of the country? Answer:'],\n",
              " [\"In which continent was the birthplace of Roma Downey's spouse? Answer: Europe. The Roma, who were once a part of the indigenous peoples of\",\n",
              "  'Which continent does the partner of Roma Downey hail from? Answer: Europe! The actress and her husband, actor-musician-director',\n",
              "  \"What is the continent of origin of Roma Downey's spouse? Answer: The couple's origins are unknown. What is the name\"],\n",
              " ['Which country does the sport played by Shaun Marcum belong to? Answer: Australia Who is the most popular player in the game of rugby league',\n",
              "  'From which country does the sport played by Shaun Marcum originate? Answer: It was invented in England, by a man by the name of John Marc',\n",
              "  'What is the country of origin for the sport in which Shaun Marcum played? Answer: Australia What is the country in which Shaun Marcum'],\n",
              " ['Who is the head of government of the country where the sport associated with Mark Lemke originated? Answer: The head of government is the head of state',\n",
              "  'What is the name of the current head of government of the country where the sport played by Mark Lemke was created? Answer: The current head of',\n",
              "  \"Which person is the head of government of the country that Mark Lemke's sport originates from? Answer: Mark Lemke is the head of government\"],\n",
              " ['Who is the head of state of the country of origin of the Eredivisie sport? Answer: The head of State of the country of',\n",
              "  \"In which country is the head of state of the Eredivisie's country of origin located? Answer: Netherlands. How many Dutch\",\n",
              "  'What is the name of the chief executive of the country where the Eredivisie sport originated? Answer: Eredivisie.\\n'],\n",
              " ['In which country was the sport played by Nigel de Jong originated? Answer: France. Who is the best player in history to have played for',\n",
              "  \"Which country is the sport associated with Nigel de Jong's position played on a team from? Answer: Germany. The Dutch midfielder's father,\",\n",
              "  'What is the country of origin of the sport played by Nigel de Jong based on his position played on a team? Answer: He played on the Netherlands'],\n",
              " [\"What is the headquarters location of the institution where Jim Hawkins' creator was educated? Answer: The headquarters of the institution that educated Jim Hawkins is\",\n",
              "  'Where is the educational institution located, which the creator of Jim Hawkins attended with its headquarters? Answer: Jim Hawkins was born on the campus of the University',\n",
              "  'In what city is the institution located where the creator of Jim Hawkins was educated, and which has its headquarters at that location? Answer: The'],\n",
              " ['What is the official language of the country where the sport associated with Dave Cowens originated? Answer: English What is the official language of the',\n",
              "  'In what language are official transactions conducted in the country of origin for the sport Dave Cowens is associated with? Answer: English What does the',\n",
              "  'What is the primary language spoken in the country where the sport Dave Cowens is associated with originated? Answer: The sport originated in the United States,'],\n",
              " ['Where was the place of work of the founder of the religion with which Camillo Ruini is associated? Answer: It is not possible to know with',\n",
              "  \"What is the location where the founder of the religion Camillo Ruini was linked with worked? Answer: It's the city of Rome, in the\",\n",
              "  \"In which city did the founder of Camillo Ruini's religion work? Answer: The first of his followers, who lived in a small village in\"],\n",
              " [\"What is the work location of the establishment founded by the founder of Desmond Connell's religion? Answer: The founder's home, a house in the\",\n",
              "  'Where was the founder of Desmond Connell\\'s religion working when he/she created it? Answer: The founding prophet, or \"prophetess,\"',\n",
              "  \"In which city did the founder of Desmond Connell's religion work when coming up with it? Answer: The city of New York City,\"],\n",
              " [\"Which continent does the country of origin of Leigh Genesis F.C.'s sport belong to? Answer: Brazil. Brazil is one of the\",\n",
              "  'What continent is the country of origin of the sport played by Leigh Genesis F.C. found in? Answer: Africa (5.8% of',\n",
              "  'In which continent can the country of origin of the sport played by Leigh Genesis F.C. be located? Answer: The United Kingdom, which is'],\n",
              " ['At which location did the founder of the religion that Ignatius Kutu Acheampong was associated with work? Answer: In the area of',\n",
              "  'Where did the founder of the religion that Ignatius Kutu Acheampong followed work? Answer: He was a slave. He was a',\n",
              "  'What is the work location of the founder of the religion that Ignatius Kutu Acheampong believed in? Answer: The founder of the'],\n",
              " [\"From which country does the sport of Bert Trautmann's specialization originate? Answer: Germany (and, of course, Switzerland) The sport\",\n",
              "  'Which country is associated with the sport in which Bert Trautmann plays his specialized position? Answer: Germany. How many times has Bert Tra',\n",
              "  \"What is the country of origin for the sport that Bert Trautmann is involved in? Answer: Bert Trautmann's Sport\\n\"],\n",
              " [\"Who is the current head of state of the country where Tanduay Rhum Masters was created as a sports team? Answer: It's not the\",\n",
              "  'What is the name of the current leader of the country of origin of Tanduay Rhum Masters sports team? Answer: The team is',\n",
              "  'In which country, whose current head of state is who, was the Tanduay Rhum Masters sports team first established? Answer: The'],\n",
              " ['What is the birthplace of the person who founded the religion Ambrose is associated with? Answer: Rome Where was the first Christian martyr to be cruc',\n",
              "  \"Which city does the founder of the religion Ambrose was associated with hail from? Answer: Rome! The Church's founder and first bishop, Saint\",\n",
              "  'In what city was the person who created the religion that Ambrose is linked with born? Answer: The city of Rome. The Church Fathers\\n'],\n",
              " [\"Who is the head of state of the country where Maccabi Tel Aviv F.C.'s sport originated? Answer: It is the\",\n",
              "  'What is the name of the leader of the country where the sport of Maccabi Tel Aviv F.C. was first played? Answer: Israel',\n",
              "  'Who is the head of state of the country where the sport played by Maccabi Tel Aviv F.C. was first established? Answer: Israel'],\n",
              " ['In what language is \"Typee\", a notable work by a certain author, written in? Answer: In English! What is the difference',\n",
              "  'What language is used for the work that the author of \"Typee\" is known for? Answer: The author of this work was known for his',\n",
              "  'Which language is attributed to the most well-known work by the author who wrote \"Typee\"? Answer: English. How many words does'],\n",
              " ['Who founded the religion associated with the spouse of Elisabeth Farnese? Answer: A. The Church of Rome. The Church was founded',\n",
              "  'Which person established the religion that was followed by the partner of Elisabeth Farnese? Answer: Elisabeth Farnese, the wife of the',\n",
              "  'What is the name of the founder of the religion which the spouse of Elisabeth Farnese adhered to? Answer: The Founder was named El'],\n",
              " ['In which continent is the country of origin of Supercopa de España located? Answer: The country of origin of Supercopa',\n",
              "  'Which continent encompasses the country where Supercopa de España originates from? Answer: The Americas! The Americas How long',\n",
              "  'What is the continent of origin of the country hosting the sporting event Supercopa de España? Answer: Argentina What is the continent'],\n",
              " ['Who is the head of state of the country of origin of the West Midlands (Regional) League? Answer: The head of state of the UK',\n",
              "  'What is the name of the chief of the state where the sport played by the West Midlands (Regional) League originated? Answer: Reg',\n",
              "  'Who is the leader of the country where the sport played by the West Midlands (Regional) League was founded? Answer: The leader of the country'],\n",
              " ['In what place is the headquarters of the educational institution where the chairperson of the Scottish National Party studied? Answer:',\n",
              "  'What is the location of the educational institution attended by the chairperson of the Scottish National Party, which now serves as the base for headquarters of the institution? Answer:',\n",
              "  'Where is the headquarters located for the institution where the chairperson of the Scottish National Party had their education? Answer:']]"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ],
      "source": [
        "\n",
        "pre_update_single=[]\n",
        "pre_update_multi=[]\n",
        "for i in range(len(request)):\n",
        "    pre_update_single.append(generate_fast(model, tok, [single_hop_prompts[i]], max_out_len=30))\n",
        "    pre_update_multi.append(generate_fast(model, tok, multi_hop_prompts[i], max_out_len=30))\n",
        "pre_update_multi"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zZBfOX1IvoRA",
        "outputId": "53f1b455-8014-479d-d38d-d001c0cfde1c"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.07333333333333333"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ],
      "source": [
        "pre_accuracy=0\n",
        "n=0\n",
        "for i in range(len(request)):\n",
        "    for ans in pre_update_multi[i]:\n",
        "        n+=1\n",
        "        ground_truth = multi_hop_answers[i]\n",
        "        is_present = any(truth in ans for truth in ground_truth)\n",
        "        if is_present:\n",
        "            pre_accuracy+=1\n",
        "\n",
        "accuracy = pre_accuracy/n\n",
        "accuracy"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "e11a74c041554bdb99fe85608e8826a3",
            "d5e4c5dbb28d4c7ba7f806fa8fdd34c1",
            "7d22e9c498634c2f9c464faee374dd1a",
            "8cde2c73c71e46759e137a72ab934011",
            "8cef951d2aaf408bb60adaf0ff43f106",
            "4222230c8cd540e1be9507c54236e705",
            "50631dd33a8345fdb9206fba151b99b0",
            "9134bedbd1b147d7ac5db5d9a4b74472",
            "44a2a9e70ffe43a486a074530c2535ea",
            "9ade9b0fb3f74300b4aeca9c75704658",
            "193cb87795a24e37a4785132b933501d"
          ]
        },
        "id": "5Ae4psxzvraT",
        "outputId": "c53d590c-8334-47f4-8548-add1f10e7ef6"
      },
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Executing ROME algorithm for the update: [Saipa F.C. is associated with the sport of] -> [ basketball]\n",
            "Cached context templates ['{}', 'A new study has. {}', 'A new report from. {}', 'The U.S. {}', 'The new version of. {}', 'In the last few. {}', 'The New York Giants. {}', 'The following is a. {}', '\"The first time. {}', 'The new year is. {}', 'The New York Times. {}', '\"We have to be able to make a. {}', 'The New England Patriots have a chance to end. {}', 'The first-time director of the White House. {}', \"In the past week, we've seen a. {}\", 'In an effort to help people understand what is. {}', 'The UESPWiki – Your source for. {}', '\"We\\'ve been looking at this issue for. {}', 'In the wake of the terrorist attack in Nice. {}', \"I'm not going to lie. I'm. {}\", \"I'm sure that if you've been paying. {}\"]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Saipa F.C.\n",
            "Retrieving inverse covariance statistics for gpt2-xl @ transformer.h.17.mlp.c_proj. The result will be cached to avoid repetitive computation.\n",
            "Attempting to download gpt2-xl/wikipedia_stats/transformer.h.17.mlp.c_proj_float32_mom2_100000.npz from https://rome.baulab.info/data/stats/gpt2-xl/wikipedia_stats/transformer.h.17.mlp.c_proj_float32_mom2_100000.npz.\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 156M/156M [00:00<00:00, 170MB/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Successfully downloaded.\n",
            "Loading cached data/stats/gpt2-xl/wikipedia_stats/transformer.h.17.mlp.c_proj_float32_mom2_100000.npz\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e11a74c041554bdb99fe85608e8826a3",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/1000 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 6 | Sentence: Saipa F.C. is associated with the sport of | Token: .\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 5.424 = 5.424 + 0.0 + 0.0 avg prob of [ basketball] 0.005193458404392004\n",
            "loss 4.886 = 4.853 + 0.003 + 0.03 avg prob of [ basketball] 0.008906635455787182\n",
            "loss 2.823 = 2.761 + 0.015 + 0.047 avg prob of [ basketball] 0.07042974978685379\n",
            "loss 0.557 = 0.459 + 0.035 + 0.063 avg prob of [ basketball] 0.6396767497062683\n",
            "loss 0.394 = 0.248 + 0.067 + 0.079 avg prob of [ basketball] 0.7838620543479919\n",
            "loss 0.401 = 0.218 + 0.09 + 0.093 avg prob of [ basketball] 0.8078551888465881\n",
            "loss 0.347 = 0.144 + 0.097 + 0.106 avg prob of [ basketball] 0.8682315349578857\n",
            "loss 0.288 = 0.091 + 0.088 + 0.109 avg prob of [ basketball] 0.9140498042106628\n",
            "loss 0.245 = 0.063 + 0.073 + 0.109 avg prob of [ basketball] 0.9394906759262085\n",
            "loss 0.215 = 0.048 + 0.059 + 0.109 avg prob of [ basketball] 0.9535272121429443\n",
            "loss 0.196 = 0.039 + 0.048 + 0.109 avg prob of [ basketball] 0.9617353081703186\n",
            "loss 0.183 = 0.033 + 0.04 + 0.109 avg prob of [ basketball] 0.9671716690063477\n",
            "loss 0.173 = 0.029 + 0.035 + 0.109 avg prob of [ basketball] 0.9712550640106201\n",
            "loss 0.166 = 0.026 + 0.032 + 0.109 avg prob of [ basketball] 0.9746259450912476\n",
            "loss 0.161 = 0.023 + 0.029 + 0.109 avg prob of [ basketball] 0.9775705337524414\n",
            "loss 0.156 = 0.02 + 0.027 + 0.109 avg prob of [ basketball] 0.9801997542381287\n",
            "loss 0.151 = 0.018 + 0.025 + 0.109 avg prob of [ basketball] 0.9825312495231628\n",
            "loss 0.148 = 0.016 + 0.023 + 0.109 avg prob of [ basketball] 0.9845528602600098\n",
            "loss 0.145 = 0.014 + 0.023 + 0.109 avg prob of [ basketball] 0.9862677454948425\n",
            "loss 0.143 = 0.012 + 0.022 + 0.109 avg prob of [ basketball] 0.9877086877822876\n",
            "Delta norm: 73.48687744140625\n",
            "Change in target norm: 18.371719360351562 to 75.63917541503906 => 57.2674560546875\n",
            "Division Factor: 9.082287788391113\n",
            "Right vector norm: 8.091230392456055\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The type of music that Betty Carter plays is] -> [ instrumental rock]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Betty Carter\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 6 | Sentence: The type of music that Betty Carter plays is instrumental | Token:  Carter\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 4.912 = 4.912 + 0.0 + 0.0 avg prob of [ instrumental rock] 0.0077115255407989025\n",
            "loss 4.382 = 4.361 + 0.001 + 0.02 avg prob of [ instrumental rock] 0.013238424435257912\n",
            "loss 3.716 = 3.669 + 0.013 + 0.034 avg prob of [ instrumental rock] 0.026477020233869553\n",
            "loss 3.204 = 3.135 + 0.023 + 0.046 avg prob of [ instrumental rock] 0.044605497270822525\n",
            "loss 2.686 = 2.61 + 0.018 + 0.058 avg prob of [ instrumental rock] 0.07521766424179077\n",
            "loss 1.998 = 1.916 + 0.013 + 0.069 avg prob of [ instrumental rock] 0.15050318837165833\n",
            "loss 1.306 = 1.207 + 0.02 + 0.079 avg prob of [ instrumental rock] 0.3028714060783386\n",
            "loss 0.67 = 0.553 + 0.029 + 0.087 avg prob of [ instrumental rock] 0.5790587663650513\n",
            "loss 0.314 = 0.189 + 0.035 + 0.09 avg prob of [ instrumental rock] 0.8303006887435913\n",
            "loss 0.236 = 0.107 + 0.039 + 0.09 avg prob of [ instrumental rock] 0.8997219800949097\n",
            "loss 0.186 = 0.056 + 0.04 + 0.09 avg prob of [ instrumental rock] 0.9463911652565002\n",
            "loss 0.163 = 0.034 + 0.038 + 0.09 avg prob of [ instrumental rock] 0.9667531847953796\n",
            "loss 0.15 = 0.026 + 0.034 + 0.09 avg prob of [ instrumental rock] 0.974550724029541\n",
            "loss 0.144 = 0.024 + 0.03 + 0.09 avg prob of [ instrumental rock] 0.9765134453773499\n",
            "loss 0.139 = 0.024 + 0.025 + 0.09 avg prob of [ instrumental rock] 0.9767534136772156\n",
            "loss 0.138 = 0.026 + 0.022 + 0.09 avg prob of [ instrumental rock] 0.9746891260147095\n",
            "loss 0.138 = 0.027 + 0.021 + 0.09 avg prob of [ instrumental rock] 0.9735832214355469\n",
            "loss 0.132 = 0.022 + 0.019 + 0.09 avg prob of [ instrumental rock] 0.9779324531555176\n",
            "loss 0.125 = 0.016 + 0.019 + 0.09 avg prob of [ instrumental rock] 0.9844022393226624\n",
            "loss 0.12 = 0.011 + 0.019 + 0.09 avg prob of [ instrumental rock] 0.9890817403793335\n",
            "Delta norm: 88.6964111328125\n",
            "Change in target norm: 22.174102783203125 to 90.59648132324219 => 68.42237854003906\n",
            "Division Factor: 11.441267967224121\n",
            "Right vector norm: 7.752323150634766\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [South Atlantic League is associated with the sport of] -> [ basketball]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object South Atlantic League\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 2 | Sentence: South Atlantic League is associated with the sport of | Token:  League\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 6.095 = 6.095 + 0.0 + 0.0 avg prob of [ basketball] 0.00252925674431026\n",
            "loss 3.263 = 3.231 + 0.007 + 0.025 avg prob of [ basketball] 0.04301953315734863\n",
            "loss 1.321 = 1.257 + 0.021 + 0.042 avg prob of [ basketball] 0.3349957764148712\n",
            "loss 0.459 = 0.379 + 0.025 + 0.055 avg prob of [ basketball] 0.6969801187515259\n",
            "loss 0.367 = 0.272 + 0.026 + 0.068 avg prob of [ basketball] 0.7672496438026428\n",
            "loss 0.344 = 0.238 + 0.028 + 0.079 avg prob of [ basketball] 0.7928363680839539\n",
            "loss 0.326 = 0.209 + 0.028 + 0.089 avg prob of [ basketball] 0.8148407936096191\n",
            "loss 0.309 = 0.184 + 0.028 + 0.098 avg prob of [ basketball] 0.835249125957489\n",
            "loss 0.289 = 0.162 + 0.027 + 0.1 avg prob of [ basketball] 0.8530324697494507\n",
            "loss 0.268 = 0.142 + 0.026 + 0.1 avg prob of [ basketball] 0.8692182302474976\n",
            "loss 0.25 = 0.125 + 0.025 + 0.1 avg prob of [ basketball] 0.884020209312439\n",
            "loss 0.233 = 0.11 + 0.024 + 0.1 avg prob of [ basketball] 0.8971706628799438\n",
            "loss 0.22 = 0.097 + 0.023 + 0.1 avg prob of [ basketball] 0.908626139163971\n",
            "loss 0.208 = 0.086 + 0.023 + 0.1 avg prob of [ basketball] 0.9184892773628235\n",
            "loss 0.199 = 0.076 + 0.022 + 0.1 avg prob of [ basketball] 0.9269365072250366\n",
            "loss 0.191 = 0.069 + 0.022 + 0.1 avg prob of [ basketball] 0.9341691732406616\n",
            "loss 0.184 = 0.062 + 0.022 + 0.1 avg prob of [ basketball] 0.9403818249702454\n",
            "loss 0.179 = 0.056 + 0.022 + 0.1 avg prob of [ basketball] 0.9457474946975708\n",
            "loss 0.174 = 0.051 + 0.022 + 0.1 avg prob of [ basketball] 0.9504140615463257\n",
            "loss 0.169 = 0.047 + 0.022 + 0.1 avg prob of [ basketball] 0.9545015692710876\n",
            "Delta norm: 79.88666534423828\n",
            "Change in target norm: 19.97166633605957 to 82.47767639160156 => 62.506011962890625\n",
            "Division Factor: 9.512157440185547\n",
            "Right vector norm: 8.398375511169434\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Jean Bodin is affiliated with the religion of] -> [ Buddhism]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Jean Bodin\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 2 | Sentence: Jean Bodin is affiliated with the religion of | Token: in\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 5.324 = 5.324 + 0.0 + 0.0 avg prob of [ Buddhism] 0.006125802174210548\n",
            "loss 3.721 = 3.687 + 0.008 + 0.026 avg prob of [ Buddhism] 0.0322316475212574\n",
            "loss 1.762 = 1.691 + 0.031 + 0.041 avg prob of [ Buddhism] 0.18820585310459137\n",
            "loss 1.128 = 1.043 + 0.03 + 0.055 avg prob of [ Buddhism] 0.3597891628742218\n",
            "loss 0.514 = 0.419 + 0.027 + 0.067 avg prob of [ Buddhism] 0.6618544459342957\n",
            "loss 0.262 = 0.157 + 0.026 + 0.079 avg prob of [ Buddhism] 0.8564138412475586\n",
            "loss 0.193 = 0.078 + 0.025 + 0.09 avg prob of [ Buddhism] 0.9255197048187256\n",
            "loss 0.172 = 0.047 + 0.025 + 0.1 avg prob of [ Buddhism] 0.9545327425003052\n",
            "loss 0.159 = 0.031 + 0.025 + 0.103 avg prob of [ Buddhism] 0.96917724609375\n",
            "loss 0.15 = 0.023 + 0.024 + 0.103 avg prob of [ Buddhism] 0.9773645401000977\n",
            "loss 0.144 = 0.018 + 0.024 + 0.103 avg prob of [ Buddhism] 0.9819148778915405\n",
            "loss 0.141 = 0.016 + 0.023 + 0.103 avg prob of [ Buddhism] 0.9845355749130249\n",
            "loss 0.139 = 0.014 + 0.022 + 0.103 avg prob of [ Buddhism] 0.9862171411514282\n",
            "loss 0.136 = 0.013 + 0.021 + 0.103 avg prob of [ Buddhism] 0.9874528646469116\n",
            "loss 0.135 = 0.012 + 0.02 + 0.103 avg prob of [ Buddhism] 0.988480806350708\n",
            "loss 0.133 = 0.011 + 0.02 + 0.103 avg prob of [ Buddhism] 0.9894102215766907\n",
            "loss 0.131 = 0.01 + 0.019 + 0.103 avg prob of [ Buddhism] 0.9902836084365845\n",
            "loss 0.13 = 0.009 + 0.018 + 0.103 avg prob of [ Buddhism] 0.9911108613014221\n",
            "loss 0.129 = 0.008 + 0.018 + 0.103 avg prob of [ Buddhism] 0.991887629032135\n",
            "loss 0.128 = 0.007 + 0.018 + 0.103 avg prob of [ Buddhism] 0.992606520652771\n",
            "Delta norm: 77.9930419921875\n",
            "Change in target norm: 19.498262405395508 to 79.49442291259766 => 59.99616241455078\n",
            "Division Factor: 10.976213455200195\n",
            "Right vector norm: 7.105642318725586\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The company that produced Chevrolet Cobalt is] -> [ Yakovlev]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Chevrolet Cobalt\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 6 | Sentence: The company that produced Chevrolet Cobalt is Yakov | Token: alt\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 7.387 = 7.387 + 0.0 + 0.0 avg prob of [ Yakovlev] 0.0006690688896924257\n",
            "loss 6.079 = 6.045 + 0.006 + 0.028 avg prob of [ Yakovlev] 0.002957097254693508\n",
            "loss 4.482 = 4.386 + 0.051 + 0.045 avg prob of [ Yakovlev] 0.015407497063279152\n",
            "loss 3.629 = 3.493 + 0.075 + 0.06 avg prob of [ Yakovlev] 0.036311738193035126\n",
            "loss 2.928 = 2.761 + 0.092 + 0.074 avg prob of [ Yakovlev] 0.071554996073246\n",
            "loss 2.296 = 2.104 + 0.105 + 0.088 avg prob of [ Yakovlev] 0.12827672064304352\n",
            "loss 1.769 = 1.559 + 0.11 + 0.101 avg prob of [ Yakovlev] 0.21202319860458374\n",
            "loss 1.381 = 1.162 + 0.112 + 0.106 avg prob of [ Yakovlev] 0.3140958547592163\n",
            "loss 1.074 = 0.856 + 0.112 + 0.106 avg prob of [ Yakovlev] 0.42722275853157043\n",
            "loss 0.81 = 0.595 + 0.108 + 0.106 avg prob of [ Yakovlev] 0.5546955466270447\n",
            "loss 0.544 = 0.338 + 0.099 + 0.106 avg prob of [ Yakovlev] 0.7163243889808655\n",
            "loss 0.34 = 0.146 + 0.087 + 0.106 avg prob of [ Yakovlev] 0.8652476668357849\n",
            "loss 0.248 = 0.061 + 0.08 + 0.106 avg prob of [ Yakovlev] 0.9407263398170471\n",
            "loss 0.219 = 0.035 + 0.077 + 0.106 avg prob of [ Yakovlev] 0.9652520418167114\n",
            "loss 0.211 = 0.029 + 0.076 + 0.106 avg prob of [ Yakovlev] 0.9713828563690186\n",
            "loss 0.208 = 0.028 + 0.074 + 0.106 avg prob of [ Yakovlev] 0.9723780751228333\n",
            "loss 0.206 = 0.028 + 0.071 + 0.106 avg prob of [ Yakovlev] 0.9723576307296753\n",
            "loss 0.203 = 0.028 + 0.069 + 0.106 avg prob of [ Yakovlev] 0.9726777076721191\n",
            "loss 0.2 = 0.027 + 0.066 + 0.106 avg prob of [ Yakovlev] 0.9737525582313538\n",
            "loss 0.195 = 0.025 + 0.064 + 0.106 avg prob of [ Yakovlev] 0.9756112694740295\n",
            "Delta norm: 75.13248443603516\n",
            "Change in target norm: 18.78312110900879 to 78.07622528076172 => 59.29310607910156\n",
            "Division Factor: 10.394453048706055\n",
            "Right vector norm: 7.228132724761963\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Louis XV of France is affiliated with the religion of] -> [ Methodism]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Louis XV of France\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 3 | Sentence: Louis XV of France is affiliated with the religion of Method | Token:  France\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 4.775 = 4.775 + 0.0 + 0.0 avg prob of [ Methodism] 0.009159033186733723\n",
            "loss 4.322 = 4.292 + 0.0 + 0.029 avg prob of [ Methodism] 0.01503460668027401\n",
            "loss 3.713 = 3.66 + 0.001 + 0.052 avg prob of [ Methodism] 0.028522366657853127\n",
            "loss 2.883 = 2.808 + 0.002 + 0.072 avg prob of [ Methodism] 0.06488627195358276\n",
            "loss 1.93 = 1.833 + 0.006 + 0.092 avg prob of [ Methodism] 0.16529420018196106\n",
            "loss 1.072 = 0.954 + 0.01 + 0.108 avg prob of [ Methodism] 0.389567106962204\n",
            "loss 0.65 = 0.53 + 0.012 + 0.108 avg prob of [ Methodism] 0.5933345556259155\n",
            "loss 0.393 = 0.272 + 0.013 + 0.108 avg prob of [ Methodism] 0.7654921412467957\n",
            "loss 0.228 = 0.102 + 0.017 + 0.108 avg prob of [ Methodism] 0.9044971466064453\n",
            "loss 0.191 = 0.055 + 0.027 + 0.108 avg prob of [ Methodism] 0.9475139379501343\n",
            "loss 0.167 = 0.032 + 0.026 + 0.108 avg prob of [ Methodism] 0.9686828255653381\n",
            "loss 0.151 = 0.021 + 0.021 + 0.108 avg prob of [ Methodism] 0.9792507886886597\n",
            "loss 0.141 = 0.016 + 0.016 + 0.108 avg prob of [ Methodism] 0.983920156955719\n",
            "loss 0.134 = 0.014 + 0.012 + 0.108 avg prob of [ Methodism] 0.9863489270210266\n",
            "loss 0.131 = 0.012 + 0.01 + 0.108 avg prob of [ Methodism] 0.9878340363502502\n",
            "loss 0.128 = 0.011 + 0.009 + 0.108 avg prob of [ Methodism] 0.9890024065971375\n",
            "loss 0.126 = 0.01 + 0.008 + 0.108 avg prob of [ Methodism] 0.990119218826294\n",
            "loss 0.125 = 0.009 + 0.008 + 0.108 avg prob of [ Methodism] 0.9912336468696594\n",
            "loss 0.123 = 0.008 + 0.007 + 0.108 avg prob of [ Methodism] 0.9923244714736938\n",
            "loss 0.122 = 0.007 + 0.007 + 0.108 avg prob of [ Methodism] 0.9933391809463501\n",
            "Delta norm: 73.95010375976562\n",
            "Change in target norm: 18.487525939941406 to 75.70298767089844 => 57.21546173095703\n",
            "Division Factor: 7.840753555297852\n",
            "Right vector norm: 9.43150520324707\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [George Orwell died in the city of] -> [ Bucharest]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object George Orwell\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 1 | Sentence: George Orwell died in the city of Buch | Token:  Orwell\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 5.434 = 5.434 + 0.0 + 0.0 avg prob of [ Bucharest] 0.004712599329650402\n",
            "loss 3.517 = 3.491 + 0.001 + 0.024 avg prob of [ Bucharest] 0.03222014382481575\n",
            "loss 2.43 = 2.384 + 0.003 + 0.042 avg prob of [ Bucharest] 0.09639173746109009\n",
            "loss 1.36 = 1.296 + 0.005 + 0.058 avg prob of [ Bucharest] 0.28339090943336487\n",
            "loss 0.549 = 0.469 + 0.007 + 0.073 avg prob of [ Bucharest] 0.6361327171325684\n",
            "loss 0.227 = 0.133 + 0.008 + 0.087 avg prob of [ Bucharest] 0.8777696490287781\n",
            "loss 0.155 = 0.046 + 0.01 + 0.099 avg prob of [ Bucharest] 0.955514669418335\n",
            "loss 0.129 = 0.021 + 0.009 + 0.099 avg prob of [ Bucharest] 0.9792338013648987\n",
            "loss 0.118 = 0.01 + 0.008 + 0.099 avg prob of [ Bucharest] 0.9895908236503601\n",
            "loss 0.113 = 0.006 + 0.008 + 0.099 avg prob of [ Bucharest] 0.9940679669380188\n",
            "loss 0.11 = 0.004 + 0.007 + 0.099 avg prob of [ Bucharest] 0.9960947036743164\n",
            "loss 0.109 = 0.003 + 0.007 + 0.099 avg prob of [ Bucharest] 0.9970735907554626\n",
            "loss 0.108 = 0.002 + 0.006 + 0.099 avg prob of [ Bucharest] 0.9975754022598267\n",
            "loss 0.107 = 0.002 + 0.006 + 0.099 avg prob of [ Bucharest] 0.9978443384170532\n",
            "loss 0.107 = 0.002 + 0.006 + 0.099 avg prob of [ Bucharest] 0.9979939460754395\n",
            "loss 0.106 = 0.002 + 0.006 + 0.099 avg prob of [ Bucharest] 0.9980809688568115\n",
            "loss 0.106 = 0.002 + 0.005 + 0.099 avg prob of [ Bucharest] 0.9981358051300049\n",
            "loss 0.106 = 0.002 + 0.005 + 0.099 avg prob of [ Bucharest] 0.9981759786605835\n",
            "loss 0.106 = 0.002 + 0.005 + 0.099 avg prob of [ Bucharest] 0.9982112646102905\n",
            "loss 0.106 = 0.002 + 0.005 + 0.099 avg prob of [ Bucharest] 0.9982476830482483\n",
            "Delta norm: 80.85773468017578\n",
            "Change in target norm: 20.214433670043945 to 83.41143798828125 => 63.19700622558594\n",
            "Division Factor: 9.150350570678711\n",
            "Right vector norm: 8.836572647094727\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [midfielder is associated with the sport of] -> [ Gaelic football]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object midfielder\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 2 | Sentence: midfielder is associated with the sport of Gaelic | Token: er\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 3.093 = 3.093 + 0.0 + 0.0 avg prob of [ Gaelic football] 0.04668448865413666\n",
            "loss 2.706 = 2.67 + 0.004 + 0.032 avg prob of [ Gaelic football] 0.07070920616388321\n",
            "loss 2.404 = 2.345 + 0.005 + 0.054 avg prob of [ Gaelic football] 0.09803888946771622\n",
            "loss 2.03 = 1.949 + 0.009 + 0.072 avg prob of [ Gaelic football] 0.1457705944776535\n",
            "loss 1.473 = 1.368 + 0.016 + 0.089 avg prob of [ Gaelic football] 0.25963306427001953\n",
            "loss 0.821 = 0.691 + 0.023 + 0.106 avg prob of [ Gaelic football] 0.5075582265853882\n",
            "loss 0.485 = 0.344 + 0.027 + 0.114 avg prob of [ Gaelic football] 0.7144622802734375\n",
            "loss 0.363 = 0.223 + 0.027 + 0.114 avg prob of [ Gaelic football] 0.8033139705657959\n",
            "loss 0.291 = 0.151 + 0.026 + 0.114 avg prob of [ Gaelic football] 0.8609873652458191\n",
            "loss 0.25 = 0.111 + 0.026 + 0.114 avg prob of [ Gaelic football] 0.8955994844436646\n",
            "loss 0.228 = 0.089 + 0.024 + 0.114 avg prob of [ Gaelic football] 0.9149632453918457\n",
            "loss 0.213 = 0.076 + 0.023 + 0.114 avg prob of [ Gaelic football] 0.9267697930335999\n",
            "loss 0.202 = 0.068 + 0.021 + 0.114 avg prob of [ Gaelic football] 0.9345641732215881\n",
            "loss 0.195 = 0.062 + 0.019 + 0.114 avg prob of [ Gaelic football] 0.9400201439857483\n",
            "loss 0.189 = 0.058 + 0.017 + 0.114 avg prob of [ Gaelic football] 0.9442331194877625\n",
            "loss 0.183 = 0.053 + 0.016 + 0.114 avg prob of [ Gaelic football] 0.948058009147644\n",
            "loss 0.178 = 0.049 + 0.015 + 0.114 avg prob of [ Gaelic football] 0.9522231221199036\n",
            "loss 0.172 = 0.044 + 0.014 + 0.114 avg prob of [ Gaelic football] 0.9571849703788757\n",
            "loss 0.166 = 0.038 + 0.013 + 0.114 avg prob of [ Gaelic football] 0.9626370668411255\n",
            "loss 0.16 = 0.033 + 0.013 + 0.114 avg prob of [ Gaelic football] 0.967741847038269\n",
            "Delta norm: 70.24609375\n",
            "Change in target norm: 17.5615234375 to 71.01618957519531 => 53.45466613769531\n",
            "Division Factor: 2.936206102371216\n",
            "Right vector norm: 23.924100875854492\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Katy Perry is a citizen of] -> [ United Kingdom]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Katy Perry\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 2 | Sentence: Katy Perry is a citizen of United | Token:  Perry\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 4.566 = 4.566 + 0.0 + 0.0 avg prob of [ United Kingdom] 0.011501044034957886\n",
            "loss 3.894 = 3.877 + 0.001 + 0.016 avg prob of [ United Kingdom] 0.023012278601527214\n",
            "loss 3.283 = 3.252 + 0.005 + 0.027 avg prob of [ United Kingdom] 0.04335806518793106\n",
            "loss 2.254 = 2.196 + 0.021 + 0.037 avg prob of [ United Kingdom] 0.1200605109333992\n",
            "loss 1.365 = 1.222 + 0.097 + 0.046 avg prob of [ United Kingdom] 0.3084986209869385\n",
            "loss 0.675 = 0.508 + 0.113 + 0.054 avg prob of [ United Kingdom] 0.6151532530784607\n",
            "loss 0.322 = 0.15 + 0.111 + 0.061 avg prob of [ United Kingdom] 0.8658409118652344\n",
            "loss 0.204 = 0.039 + 0.098 + 0.067 avg prob of [ United Kingdom] 0.9625231623649597\n",
            "loss 0.167 = 0.016 + 0.078 + 0.073 avg prob of [ United Kingdom] 0.9846287965774536\n",
            "loss 0.154 = 0.01 + 0.065 + 0.079 avg prob of [ United Kingdom] 0.9899544715881348\n",
            "loss 0.145 = 0.009 + 0.057 + 0.079 avg prob of [ United Kingdom] 0.9912396669387817\n",
            "loss 0.137 = 0.008 + 0.05 + 0.079 avg prob of [ United Kingdom] 0.992127001285553\n",
            "loss 0.127 = 0.007 + 0.041 + 0.079 avg prob of [ United Kingdom] 0.9929397702217102\n",
            "loss 0.117 = 0.006 + 0.032 + 0.079 avg prob of [ United Kingdom] 0.9937604665756226\n",
            "loss 0.109 = 0.005 + 0.025 + 0.079 avg prob of [ United Kingdom] 0.9945791363716125\n",
            "loss 0.107 = 0.005 + 0.023 + 0.079 avg prob of [ United Kingdom] 0.9953479170799255\n",
            "loss 0.105 = 0.004 + 0.023 + 0.079 avg prob of [ United Kingdom] 0.9960205554962158\n",
            "loss 0.102 = 0.003 + 0.02 + 0.079 avg prob of [ United Kingdom] 0.9965847134590149\n",
            "loss 0.098 = 0.003 + 0.016 + 0.079 avg prob of [ United Kingdom] 0.997052788734436\n",
            "loss 0.095 = 0.003 + 0.013 + 0.079 avg prob of [ United Kingdom] 0.9974387288093567\n",
            "Delta norm: 101.38500213623047\n",
            "Change in target norm: 25.346248626708984 to 105.34003448486328 => 79.99378967285156\n",
            "Division Factor: 9.301328659057617\n",
            "Right vector norm: 10.900054931640625\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Basil D'Oliveira is associated with the sport of] -> [ basketball]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Basil D'Oliveira\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 6 | Sentence: Basil D'Oliveira is associated with the sport of | Token: ira\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 5.802 = 5.802 + 0.0 + 0.0 avg prob of [ basketball] 0.003870752640068531\n",
            "loss 4.114 = 4.089 + 0.004 + 0.021 avg prob of [ basketball] 0.021629061549901962\n",
            "loss 1.989 = 1.949 + 0.005 + 0.035 avg prob of [ basketball] 0.20847280323505402\n",
            "loss 0.437 = 0.381 + 0.008 + 0.047 avg prob of [ basketball] 0.7139735817909241\n",
            "loss 0.228 = 0.159 + 0.012 + 0.057 avg prob of [ basketball] 0.8550564050674438\n",
            "loss 0.199 = 0.118 + 0.015 + 0.067 avg prob of [ basketball] 0.8899489045143127\n",
            "loss 0.195 = 0.102 + 0.017 + 0.076 avg prob of [ basketball] 0.9037255048751831\n",
            "loss 0.192 = 0.09 + 0.019 + 0.083 avg prob of [ basketball] 0.9143650531768799\n",
            "loss 0.188 = 0.079 + 0.02 + 0.09 avg prob of [ basketball] 0.9246435761451721\n",
            "loss 0.179 = 0.067 + 0.02 + 0.093 avg prob of [ basketball] 0.9353678822517395\n",
            "loss 0.168 = 0.056 + 0.019 + 0.093 avg prob of [ basketball] 0.9453005194664001\n",
            "loss 0.159 = 0.048 + 0.019 + 0.093 avg prob of [ basketball] 0.9534289240837097\n",
            "loss 0.151 = 0.041 + 0.018 + 0.093 avg prob of [ basketball] 0.9599334597587585\n",
            "loss 0.145 = 0.036 + 0.017 + 0.093 avg prob of [ basketball] 0.9650990962982178\n",
            "loss 0.14 = 0.031 + 0.016 + 0.093 avg prob of [ basketball] 0.9692081212997437\n",
            "loss 0.136 = 0.028 + 0.016 + 0.093 avg prob of [ basketball] 0.9725027680397034\n",
            "loss 0.133 = 0.025 + 0.015 + 0.093 avg prob of [ basketball] 0.9751752614974976\n",
            "loss 0.13 = 0.023 + 0.015 + 0.093 avg prob of [ basketball] 0.9773744344711304\n",
            "loss 0.128 = 0.021 + 0.014 + 0.093 avg prob of [ basketball] 0.9792142510414124\n",
            "loss 0.126 = 0.019 + 0.014 + 0.093 avg prob of [ basketball] 0.9807804822921753\n",
            "Delta norm: 86.38005065917969\n",
            "Change in target norm: 21.595014572143555 to 91.02330017089844 => 69.42828369140625\n",
            "Division Factor: 9.784863471984863\n",
            "Right vector norm: 8.827926635742188\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Abet Guidaben plays the position of] -> [ infielder]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Abet Guidaben\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 4 | Sentence: Abet Guidaben plays the position of infield | Token: en\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 5.189 = 5.189 + 0.0 + 0.0 avg prob of [ infielder] 0.007507388014346361\n",
            "loss 4.644 = 4.612 + 0.011 + 0.02 avg prob of [ infielder] 0.014156321994960308\n",
            "loss 3.744 = 3.708 + 0.004 + 0.032 avg prob of [ infielder] 0.039621978998184204\n",
            "loss 2.325 = 2.271 + 0.012 + 0.042 avg prob of [ infielder] 0.11034979671239853\n",
            "loss 2.082 = 2.009 + 0.021 + 0.052 avg prob of [ infielder] 0.14035579562187195\n",
            "loss 1.802 = 1.716 + 0.026 + 0.06 avg prob of [ infielder] 0.18602141737937927\n",
            "loss 1.426 = 1.329 + 0.029 + 0.069 avg prob of [ infielder] 0.27098020911216736\n",
            "loss 1.011 = 0.905 + 0.03 + 0.076 avg prob of [ infielder] 0.41036784648895264\n",
            "loss 0.649 = 0.537 + 0.029 + 0.083 avg prob of [ infielder] 0.5888186693191528\n",
            "loss 0.432 = 0.312 + 0.029 + 0.09 avg prob of [ infielder] 0.7345179915428162\n",
            "loss 0.332 = 0.212 + 0.03 + 0.09 avg prob of [ infielder] 0.8112175464630127\n",
            "loss 0.276 = 0.152 + 0.034 + 0.09 avg prob of [ infielder] 0.8609955310821533\n",
            "loss 0.233 = 0.102 + 0.04 + 0.09 avg prob of [ infielder] 0.9040502905845642\n",
            "loss 0.192 = 0.059 + 0.043 + 0.09 avg prob of [ infielder] 0.943510890007019\n",
            "loss 0.157 = 0.03 + 0.037 + 0.09 avg prob of [ infielder] 0.9711069464683533\n",
            "loss 0.137 = 0.016 + 0.031 + 0.09 avg prob of [ infielder] 0.9842504262924194\n",
            "loss 0.129 = 0.011 + 0.028 + 0.09 avg prob of [ infielder] 0.9892163872718811\n",
            "loss 0.126 = 0.009 + 0.027 + 0.09 avg prob of [ infielder] 0.991093635559082\n",
            "loss 0.125 = 0.008 + 0.027 + 0.09 avg prob of [ infielder] 0.9919816851615906\n",
            "loss 0.124 = 0.007 + 0.027 + 0.09 avg prob of [ infielder] 0.9925633668899536\n",
            "Delta norm: 88.64411926269531\n",
            "Change in target norm: 22.161029815673828 to 91.92150115966797 => 69.76046752929688\n",
            "Division Factor: 10.392997741699219\n",
            "Right vector norm: 8.529214859008789\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Grégory Coupet plays the position of] -> [ defenceman]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Grégory Coupet\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 5 | Sentence: Grégory Coupet plays the position of defence | Token: pet\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 3.651 = 3.651 + 0.0 + 0.0 avg prob of [ defenceman] 0.03989342972636223\n",
            "loss 2.884 = 2.858 + 0.005 + 0.021 avg prob of [ defenceman] 0.08283691108226776\n",
            "loss 2.05 = 2.001 + 0.013 + 0.036 avg prob of [ defenceman] 0.16854722797870636\n",
            "loss 1.574 = 1.507 + 0.02 + 0.047 avg prob of [ defenceman] 0.2556891441345215\n",
            "loss 0.973 = 0.896 + 0.02 + 0.057 avg prob of [ defenceman] 0.45745065808296204\n",
            "loss 0.532 = 0.444 + 0.021 + 0.066 avg prob of [ defenceman] 0.6849771738052368\n",
            "loss 0.256 = 0.148 + 0.031 + 0.076 avg prob of [ defenceman] 0.8710348010063171\n",
            "loss 0.158 = 0.04 + 0.032 + 0.086 avg prob of [ defenceman] 0.9611921906471252\n",
            "loss 0.142 = 0.017 + 0.033 + 0.092 avg prob of [ defenceman] 0.9835299849510193\n",
            "loss 0.13 = 0.01 + 0.028 + 0.092 avg prob of [ defenceman] 0.9905182123184204\n",
            "loss 0.124 = 0.006 + 0.025 + 0.092 avg prob of [ defenceman] 0.993984580039978\n",
            "loss 0.119 = 0.004 + 0.023 + 0.092 avg prob of [ defenceman] 0.995642900466919\n",
            "loss 0.116 = 0.004 + 0.02 + 0.092 avg prob of [ defenceman] 0.9963357448577881\n",
            "loss 0.114 = 0.003 + 0.018 + 0.092 avg prob of [ defenceman] 0.9965344667434692\n",
            "loss 0.114 = 0.003 + 0.018 + 0.092 avg prob of [ defenceman] 0.9965702295303345\n",
            "loss 0.112 = 0.003 + 0.016 + 0.092 avg prob of [ defenceman] 0.9966732263565063\n",
            "loss 0.111 = 0.003 + 0.016 + 0.092 avg prob of [ defenceman] 0.9968892335891724\n",
            "loss 0.111 = 0.003 + 0.016 + 0.092 avg prob of [ defenceman] 0.9971712231636047\n",
            "loss 0.11 = 0.003 + 0.015 + 0.092 avg prob of [ defenceman] 0.9974743127822876\n",
            "loss 0.109 = 0.002 + 0.015 + 0.092 avg prob of [ defenceman] 0.9977659583091736\n",
            "Delta norm: 86.57042694091797\n",
            "Change in target norm: 21.642606735229492 to 90.6878890991211 => 69.04528045654297\n",
            "Division Factor: 12.020621299743652\n",
            "Right vector norm: 7.201826572418213\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The type of music that Mike Stern plays is] -> [ country music]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Mike Stern\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 6 | Sentence: The type of music that Mike Stern plays is country | Token:  Stern\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 4.843 = 4.843 + 0.0 + 0.0 avg prob of [ country music] 0.008959705010056496\n",
            "loss 4.29 = 4.271 + 0.001 + 0.018 avg prob of [ country music] 0.01598963513970375\n",
            "loss 3.424 = 3.384 + 0.009 + 0.031 avg prob of [ country music] 0.03853921592235565\n",
            "loss 2.071 = 2.012 + 0.018 + 0.041 avg prob of [ country music] 0.13881371915340424\n",
            "loss 0.998 = 0.916 + 0.03 + 0.051 avg prob of [ country music] 0.4052838385105133\n",
            "loss 0.527 = 0.424 + 0.042 + 0.061 avg prob of [ country music] 0.6572450399398804\n",
            "loss 0.346 = 0.225 + 0.05 + 0.071 avg prob of [ country music] 0.7995402216911316\n",
            "loss 0.266 = 0.134 + 0.053 + 0.08 avg prob of [ country music] 0.8752356171607971\n",
            "loss 0.224 = 0.089 + 0.051 + 0.085 avg prob of [ country music] 0.9152178168296814\n",
            "loss 0.197 = 0.065 + 0.047 + 0.085 avg prob of [ country music] 0.9370290637016296\n",
            "loss 0.175 = 0.048 + 0.043 + 0.085 avg prob of [ country music] 0.9532943367958069\n",
            "loss 0.16 = 0.035 + 0.04 + 0.085 avg prob of [ country music] 0.9653642177581787\n",
            "loss 0.149 = 0.026 + 0.039 + 0.085 avg prob of [ country music] 0.9742230176925659\n",
            "loss 0.142 = 0.02 + 0.038 + 0.085 avg prob of [ country music] 0.980648398399353\n",
            "loss 0.137 = 0.015 + 0.038 + 0.085 avg prob of [ country music] 0.9852689504623413\n",
            "loss 0.133 = 0.011 + 0.037 + 0.085 avg prob of [ country music] 0.9885866045951843\n",
            "loss 0.129 = 0.009 + 0.036 + 0.085 avg prob of [ country music] 0.9909895062446594\n",
            "loss 0.127 = 0.007 + 0.035 + 0.085 avg prob of [ country music] 0.9927724599838257\n",
            "loss 0.125 = 0.006 + 0.035 + 0.085 avg prob of [ country music] 0.994152843952179\n",
            "loss 0.123 = 0.005 + 0.034 + 0.085 avg prob of [ country music] 0.9952730536460876\n",
            "Delta norm: 94.60185241699219\n",
            "Change in target norm: 23.650461196899414 to 99.3405532836914 => 75.69009399414062\n",
            "Division Factor: 9.309310913085938\n",
            "Right vector norm: 10.162068367004395\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Louis of Toulouse is affiliated with the religion of] -> [ Confucianism]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Louis of Toulouse\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 4 | Sentence: Louis of Toulouse is affiliated with the religion of Confucian | Token: ouse\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 2.003 = 2.003 + 0.0 + 0.0 avg prob of [ Confucianism] 0.13731586933135986\n",
            "loss 1.755 = 1.723 + 0.008 + 0.024 avg prob of [ Confucianism] 0.18195585906505585\n",
            "loss 1.536 = 1.492 + 0.005 + 0.039 avg prob of [ Confucianism] 0.22944462299346924\n",
            "loss 1.13 = 1.069 + 0.009 + 0.052 avg prob of [ Confucianism] 0.34890374541282654\n",
            "loss 0.465 = 0.385 + 0.017 + 0.064 avg prob of [ Confucianism] 0.6844268441200256\n",
            "loss 0.196 = 0.091 + 0.029 + 0.076 avg prob of [ Confucianism] 0.9129489660263062\n",
            "loss 0.171 = 0.049 + 0.035 + 0.088 avg prob of [ Confucianism] 0.9526267647743225\n",
            "loss 0.168 = 0.037 + 0.033 + 0.098 avg prob of [ Confucianism] 0.9634448885917664\n",
            "loss 0.156 = 0.03 + 0.028 + 0.098 avg prob of [ Confucianism] 0.9700835347175598\n",
            "loss 0.145 = 0.024 + 0.024 + 0.098 avg prob of [ Confucianism] 0.9766411781311035\n",
            "loss 0.136 = 0.018 + 0.02 + 0.098 avg prob of [ Confucianism] 0.9822031855583191\n",
            "loss 0.129 = 0.014 + 0.017 + 0.098 avg prob of [ Confucianism] 0.9863405227661133\n",
            "loss 0.125 = 0.011 + 0.016 + 0.098 avg prob of [ Confucianism] 0.989189863204956\n",
            "loss 0.124 = 0.009 + 0.017 + 0.098 avg prob of [ Confucianism] 0.9911119341850281\n",
            "loss 0.122 = 0.008 + 0.016 + 0.098 avg prob of [ Confucianism] 0.9924365878105164\n",
            "loss 0.119 = 0.007 + 0.014 + 0.098 avg prob of [ Confucianism] 0.9933862686157227\n",
            "loss 0.117 = 0.006 + 0.013 + 0.098 avg prob of [ Confucianism] 0.9940965175628662\n",
            "loss 0.115 = 0.005 + 0.012 + 0.098 avg prob of [ Confucianism] 0.9946487545967102\n",
            "loss 0.114 = 0.005 + 0.011 + 0.098 avg prob of [ Confucianism] 0.9950945973396301\n",
            "loss 0.114 = 0.005 + 0.011 + 0.098 avg prob of [ Confucianism] 0.9954698085784912\n",
            "Delta norm: 81.5532455444336\n",
            "Change in target norm: 20.388309478759766 to 85.36051940917969 => 64.97221374511719\n",
            "Division Factor: 9.576773643493652\n",
            "Right vector norm: 8.515732765197754\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Don Balón is associated with the sport of] -> [ baseball]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Don Balón\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 2 | Sentence: Don Balón is associated with the sport of | Token: ón\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 3.742 = 3.742 + 0.0 + 0.0 avg prob of [ baseball] 0.03245751932263374\n",
            "loss 1.048 = 1.02 + 0.004 + 0.024 avg prob of [ baseball] 0.46155112981796265\n",
            "loss 0.195 = 0.134 + 0.019 + 0.041 avg prob of [ baseball] 0.8772071003913879\n",
            "loss 0.171 = 0.092 + 0.023 + 0.056 avg prob of [ baseball] 0.9136326313018799\n",
            "loss 0.171 = 0.079 + 0.024 + 0.068 avg prob of [ baseball] 0.925072193145752\n",
            "loss 0.173 = 0.071 + 0.023 + 0.079 avg prob of [ baseball] 0.9315454363822937\n",
            "loss 0.174 = 0.065 + 0.021 + 0.088 avg prob of [ baseball] 0.9370605945587158\n",
            "loss 0.175 = 0.059 + 0.02 + 0.096 avg prob of [ baseball] 0.9427387714385986\n",
            "loss 0.169 = 0.053 + 0.018 + 0.099 avg prob of [ baseball] 0.9487572908401489\n",
            "loss 0.162 = 0.047 + 0.017 + 0.099 avg prob of [ baseball] 0.9544864892959595\n",
            "loss 0.156 = 0.041 + 0.016 + 0.099 avg prob of [ baseball] 0.9596773982048035\n",
            "loss 0.15 = 0.036 + 0.015 + 0.099 avg prob of [ baseball] 0.9642619490623474\n",
            "loss 0.145 = 0.032 + 0.014 + 0.099 avg prob of [ baseball] 0.9682558178901672\n",
            "loss 0.141 = 0.029 + 0.014 + 0.099 avg prob of [ baseball] 0.9717132449150085\n",
            "loss 0.138 = 0.026 + 0.013 + 0.099 avg prob of [ baseball] 0.9747005105018616\n",
            "loss 0.135 = 0.023 + 0.013 + 0.099 avg prob of [ baseball] 0.9772841930389404\n",
            "loss 0.132 = 0.021 + 0.013 + 0.099 avg prob of [ baseball] 0.9795246124267578\n",
            "loss 0.13 = 0.019 + 0.013 + 0.099 avg prob of [ baseball] 0.9814740419387817\n",
            "loss 0.128 = 0.017 + 0.013 + 0.099 avg prob of [ baseball] 0.9831762909889221\n",
            "loss 0.127 = 0.015 + 0.013 + 0.099 avg prob of [ baseball] 0.9846680164337158\n",
            "Delta norm: 81.17965698242188\n",
            "Change in target norm: 20.29491424560547 to 85.3438720703125 => 65.04895782470703\n",
            "Division Factor: 8.950776100158691\n",
            "Right vector norm: 9.06956672668457\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The head coach of Toronto Blue Jays is] -> [ Keisuke Honda]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Toronto Blue Jays\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 6 | Sentence: The head coach of Toronto Blue Jays is Keisuke | Token:  Jays\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 4.458 = 4.458 + 0.0 + 0.0 avg prob of [ Keisuke Honda] 0.01194214541465044\n",
            "loss 3.723 = 3.69 + 0.004 + 0.029 avg prob of [ Keisuke Honda] 0.026219729334115982\n",
            "loss 2.86 = 2.803 + 0.007 + 0.05 avg prob of [ Keisuke Honda] 0.06417793035507202\n",
            "loss 2.204 = 2.124 + 0.01 + 0.07 avg prob of [ Keisuke Honda] 0.1223195418715477\n",
            "loss 1.598 = 1.496 + 0.014 + 0.088 avg prob of [ Keisuke Honda] 0.2253866195678711\n",
            "loss 1.054 = 0.926 + 0.023 + 0.105 avg prob of [ Keisuke Honda] 0.4001472592353821\n",
            "loss 0.662 = 0.526 + 0.028 + 0.108 avg prob of [ Keisuke Honda] 0.5947457551956177\n",
            "loss 0.356 = 0.217 + 0.03 + 0.108 avg prob of [ Keisuke Honda] 0.8071717619895935\n",
            "loss 0.188 = 0.046 + 0.033 + 0.108 avg prob of [ Keisuke Honda] 0.954839289188385\n",
            "loss 0.161 = 0.017 + 0.035 + 0.108 avg prob of [ Keisuke Honda] 0.9827257990837097\n",
            "loss 0.152 = 0.012 + 0.032 + 0.108 avg prob of [ Keisuke Honda] 0.988198459148407\n",
            "loss 0.144 = 0.01 + 0.026 + 0.108 avg prob of [ Keisuke Honda] 0.9904089570045471\n",
            "loss 0.138 = 0.008 + 0.021 + 0.108 avg prob of [ Keisuke Honda] 0.9918580055236816\n",
            "loss 0.133 = 0.007 + 0.017 + 0.108 avg prob of [ Keisuke Honda] 0.992959201335907\n",
            "loss 0.129 = 0.006 + 0.015 + 0.108 avg prob of [ Keisuke Honda] 0.9938173890113831\n",
            "loss 0.127 = 0.006 + 0.013 + 0.108 avg prob of [ Keisuke Honda] 0.9944965243339539\n",
            "loss 0.127 = 0.005 + 0.014 + 0.108 avg prob of [ Keisuke Honda] 0.9950641393661499\n",
            "loss 0.126 = 0.004 + 0.013 + 0.108 avg prob of [ Keisuke Honda] 0.9955900311470032\n",
            "loss 0.124 = 0.004 + 0.012 + 0.108 avg prob of [ Keisuke Honda] 0.9961137771606445\n",
            "loss 0.123 = 0.003 + 0.011 + 0.108 avg prob of [ Keisuke Honda] 0.9966239929199219\n",
            "Delta norm: 73.85289001464844\n",
            "Change in target norm: 18.46322250366211 to 76.94781494140625 => 58.48459243774414\n",
            "Division Factor: 8.958906173706055\n",
            "Right vector norm: 8.24351692199707\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [William James is employed by] -> [ Johns Hopkins University]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object William James\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 1 | Sentence: William James is employed by Johns Hopkins | Token:  James\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 2.781 = 2.781 + 0.0 + 0.0 avg prob of [ Johns Hopkins University] 0.06578826904296875\n",
            "loss 1.712 = 1.673 + 0.024 + 0.015 avg prob of [ Johns Hopkins University] 0.1924913078546524\n",
            "loss 1.098 = 1.039 + 0.034 + 0.025 avg prob of [ Johns Hopkins University] 0.3609764873981476\n",
            "loss 0.539 = 0.47 + 0.036 + 0.034 avg prob of [ Johns Hopkins University] 0.6278758645057678\n",
            "loss 0.299 = 0.219 + 0.039 + 0.042 avg prob of [ Johns Hopkins University] 0.8046050667762756\n",
            "loss 0.239 = 0.149 + 0.041 + 0.049 avg prob of [ Johns Hopkins University] 0.8620671033859253\n",
            "loss 0.21 = 0.113 + 0.041 + 0.056 avg prob of [ Johns Hopkins University] 0.8935998678207397\n",
            "loss 0.189 = 0.086 + 0.041 + 0.062 avg prob of [ Johns Hopkins University] 0.9175298810005188\n",
            "loss 0.174 = 0.066 + 0.041 + 0.067 avg prob of [ Johns Hopkins University] 0.9365237951278687\n",
            "loss 0.164 = 0.05 + 0.042 + 0.072 avg prob of [ Johns Hopkins University] 0.9513905048370361\n",
            "loss 0.157 = 0.038 + 0.042 + 0.077 avg prob of [ Johns Hopkins University] 0.9627174139022827\n",
            "loss 0.149 = 0.03 + 0.041 + 0.079 avg prob of [ Johns Hopkins University] 0.9704554677009583\n",
            "loss 0.142 = 0.024 + 0.039 + 0.079 avg prob of [ Johns Hopkins University] 0.9759425520896912\n",
            "loss 0.135 = 0.02 + 0.037 + 0.079 avg prob of [ Johns Hopkins University] 0.980200469493866\n",
            "loss 0.13 = 0.017 + 0.035 + 0.079 avg prob of [ Johns Hopkins University] 0.9834958910942078\n",
            "loss 0.126 = 0.014 + 0.033 + 0.079 avg prob of [ Johns Hopkins University] 0.9860514998435974\n",
            "loss 0.123 = 0.012 + 0.033 + 0.079 avg prob of [ Johns Hopkins University] 0.9880431890487671\n",
            "loss 0.121 = 0.01 + 0.032 + 0.079 avg prob of [ Johns Hopkins University] 0.989606499671936\n",
            "loss 0.119 = 0.009 + 0.031 + 0.079 avg prob of [ Johns Hopkins University] 0.9908443689346313\n",
            "loss 0.117 = 0.008 + 0.031 + 0.079 avg prob of [ Johns Hopkins University] 0.9918336868286133\n",
            "Delta norm: 101.87374877929688\n",
            "Change in target norm: 25.468435287475586 to 105.61929321289062 => 80.1508560180664\n",
            "Division Factor: 11.889808654785156\n",
            "Right vector norm: 8.568156242370605\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The type of music that Sturgill Simpson plays is] -> [ reggae]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Sturgill Simpson\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 8 | Sentence: The type of music that Sturgill Simpson plays is reg | Token:  Simpson\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 3.495 = 3.495 + 0.0 + 0.0 avg prob of [ reggae] 0.03432111814618111\n",
            "loss 1.92 = 1.912 + 0.002 + 0.007 avg prob of [ reggae] 0.1522233784198761\n",
            "loss 1.156 = 1.137 + 0.008 + 0.011 avg prob of [ reggae] 0.32501110434532166\n",
            "loss 0.504 = 0.475 + 0.014 + 0.015 avg prob of [ reggae] 0.6266121864318848\n",
            "loss 0.189 = 0.156 + 0.014 + 0.018 avg prob of [ reggae] 0.8564173579216003\n",
            "loss 0.096 = 0.056 + 0.018 + 0.021 avg prob of [ reggae] 0.9452683925628662\n",
            "loss 0.08 = 0.032 + 0.024 + 0.024 avg prob of [ reggae] 0.9685937762260437\n",
            "loss 0.076 = 0.023 + 0.027 + 0.026 avg prob of [ reggae] 0.9774190187454224\n",
            "loss 0.073 = 0.018 + 0.026 + 0.028 avg prob of [ reggae] 0.9822371602058411\n",
            "loss 0.07 = 0.015 + 0.025 + 0.03 avg prob of [ reggae] 0.9855576753616333\n",
            "loss 0.068 = 0.012 + 0.024 + 0.032 avg prob of [ reggae] 0.9882313013076782\n",
            "loss 0.067 = 0.01 + 0.023 + 0.034 avg prob of [ reggae] 0.9904211163520813\n",
            "loss 0.066 = 0.008 + 0.023 + 0.035 avg prob of [ reggae] 0.9921225905418396\n",
            "loss 0.067 = 0.007 + 0.023 + 0.037 avg prob of [ reggae] 0.9933904409408569\n",
            "loss 0.067 = 0.006 + 0.023 + 0.038 avg prob of [ reggae] 0.9943257570266724\n",
            "loss 0.068 = 0.005 + 0.024 + 0.039 avg prob of [ reggae] 0.9950264692306519\n",
            "loss 0.068 = 0.004 + 0.024 + 0.04 avg prob of [ reggae] 0.9955657720565796\n",
            "loss 0.068 = 0.004 + 0.023 + 0.041 avg prob of [ reggae] 0.9959933161735535\n",
            "loss 0.068 = 0.004 + 0.023 + 0.042 avg prob of [ reggae] 0.9963415265083313\n",
            "loss 0.068 = 0.003 + 0.022 + 0.042 avg prob of [ reggae] 0.9966314435005188\n",
            "Delta norm: 120.81444549560547\n",
            "Change in target norm: 37.71969223022461 to 133.40797424316406 => 95.68827819824219\n",
            "Division Factor: 10.712958335876465\n",
            "Right vector norm: 11.277412414550781\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Anne Rice is a citizen of] -> [ Great Britain]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Anne Rice\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 1 | Sentence: Anne Rice is a citizen of Great | Token:  Rice\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 2.621 = 2.621 + 0.0 + 0.0 avg prob of [ Great Britain] 0.0791751816868782\n",
            "loss 1.356 = 1.343 + 0.004 + 0.009 avg prob of [ Great Britain] 0.26417452096939087\n",
            "loss 0.692 = 0.667 + 0.01 + 0.014 avg prob of [ Great Britain] 0.51859050989151\n",
            "loss 0.276 = 0.242 + 0.014 + 0.02 avg prob of [ Great Britain] 0.7877414226531982\n",
            "loss 0.117 = 0.079 + 0.014 + 0.024 avg prob of [ Great Britain] 0.9247224926948547\n",
            "loss 0.072 = 0.031 + 0.014 + 0.028 avg prob of [ Great Britain] 0.9699302315711975\n",
            "loss 0.061 = 0.016 + 0.013 + 0.032 avg prob of [ Great Britain] 0.9842634797096252\n",
            "loss 0.058 = 0.01 + 0.013 + 0.035 avg prob of [ Great Britain] 0.9896588325500488\n",
            "loss 0.058 = 0.008 + 0.012 + 0.038 avg prob of [ Great Britain] 0.9921867847442627\n",
            "loss 0.059 = 0.006 + 0.012 + 0.041 avg prob of [ Great Britain] 0.9936046004295349\n",
            "loss 0.06 = 0.006 + 0.012 + 0.043 avg prob of [ Great Britain] 0.9945064783096313\n",
            "loss 0.061 = 0.005 + 0.011 + 0.045 avg prob of [ Great Britain] 0.9951315522193909\n",
            "loss 0.062 = 0.004 + 0.011 + 0.047 avg prob of [ Great Britain] 0.9955924153327942\n",
            "loss 0.064 = 0.004 + 0.011 + 0.049 avg prob of [ Great Britain] 0.9959484338760376\n",
            "loss 0.065 = 0.004 + 0.011 + 0.05 avg prob of [ Great Britain] 0.9962338209152222\n",
            "loss 0.066 = 0.004 + 0.011 + 0.051 avg prob of [ Great Britain] 0.9964691400527954\n",
            "loss 0.067 = 0.003 + 0.011 + 0.053 avg prob of [ Great Britain] 0.9966678023338318\n",
            "loss 0.068 = 0.003 + 0.011 + 0.054 avg prob of [ Great Britain] 0.9968387484550476\n",
            "loss 0.068 = 0.003 + 0.01 + 0.055 avg prob of [ Great Britain] 0.9969882369041443\n",
            "loss 0.069 = 0.003 + 0.01 + 0.056 avg prob of [ Great Britain] 0.9971203804016113\n",
            "Delta norm: 128.84722900390625\n",
            "Change in target norm: 34.0146369934082 to 136.60858154296875 => 102.59394836425781\n",
            "Division Factor: 11.456953048706055\n",
            "Right vector norm: 11.246204376220703\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The author of Gesta Danorum is] -> [ Vyasa]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Gesta Danorum\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 6 | Sentence: The author of Gesta Danorum is Vy | Token: orum\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 6.195 = 6.195 + 0.0 + 0.0 avg prob of [ Vyasa] 0.002170330146327615\n",
            "loss 5.281 = 5.254 + 0.003 + 0.024 avg prob of [ Vyasa] 0.005920989904552698\n",
            "loss 3.718 = 3.671 + 0.008 + 0.039 avg prob of [ Vyasa] 0.02944916859269142\n",
            "loss 2.722 = 2.656 + 0.014 + 0.052 avg prob of [ Vyasa] 0.07742799073457718\n",
            "loss 2.279 = 2.195 + 0.02 + 0.064 avg prob of [ Vyasa] 0.11806964129209518\n",
            "loss 1.908 = 1.806 + 0.026 + 0.076 avg prob of [ Vyasa] 0.16878031194210052\n",
            "loss 1.62 = 1.499 + 0.034 + 0.087 avg prob of [ Vyasa] 0.22521404922008514\n",
            "loss 1.406 = 1.264 + 0.046 + 0.097 avg prob of [ Vyasa] 0.28348639607429504\n",
            "loss 1.251 = 1.102 + 0.051 + 0.098 avg prob of [ Vyasa] 0.3329204022884369\n",
            "loss 1.108 = 0.959 + 0.051 + 0.098 avg prob of [ Vyasa] 0.38412874937057495\n",
            "loss 0.965 = 0.82 + 0.047 + 0.098 avg prob of [ Vyasa] 0.441999226808548\n",
            "loss 0.806 = 0.665 + 0.043 + 0.098 avg prob of [ Vyasa] 0.5167129039764404\n",
            "loss 0.612 = 0.474 + 0.041 + 0.098 avg prob of [ Vyasa] 0.6260632872581482\n",
            "loss 0.408 = 0.268 + 0.043 + 0.098 avg prob of [ Vyasa] 0.7678992748260498\n",
            "loss 0.28 = 0.136 + 0.046 + 0.098 avg prob of [ Vyasa] 0.873689591884613\n",
            "loss 0.23 = 0.087 + 0.045 + 0.098 avg prob of [ Vyasa] 0.9171493053436279\n",
            "loss 0.203 = 0.066 + 0.04 + 0.098 avg prob of [ Vyasa] 0.9366974830627441\n",
            "loss 0.183 = 0.051 + 0.034 + 0.098 avg prob of [ Vyasa] 0.9501597881317139\n",
            "loss 0.171 = 0.04 + 0.033 + 0.098 avg prob of [ Vyasa] 0.9606409668922424\n",
            "loss 0.162 = 0.032 + 0.032 + 0.098 avg prob of [ Vyasa] 0.968812882900238\n",
            "Delta norm: 81.85897064208984\n",
            "Change in target norm: 20.46474266052246 to 84.14778137207031 => 63.68303680419922\n",
            "Division Factor: 10.544303894042969\n",
            "Right vector norm: 7.763335704803467\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Pedro Arrupe is affiliated with the religion of] -> [ Methodism]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Pedro Arrupe\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 4 | Sentence: Pedro Arrupe is affiliated with the religion of Method | Token: pe\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 4.538 = 4.538 + 0.0 + 0.0 avg prob of [ Methodism] 0.011212226003408432\n",
            "loss 4.078 = 4.049 + 0.012 + 0.017 avg prob of [ Methodism] 0.01873328536748886\n",
            "loss 2.796 = 2.748 + 0.021 + 0.027 avg prob of [ Methodism] 0.07016360014677048\n",
            "loss 1.271 = 1.203 + 0.032 + 0.036 avg prob of [ Methodism] 0.3096485137939453\n",
            "loss 0.432 = 0.347 + 0.042 + 0.043 avg prob of [ Methodism] 0.7130225896835327\n",
            "loss 0.237 = 0.138 + 0.049 + 0.05 avg prob of [ Methodism] 0.8725703358650208\n",
            "loss 0.161 = 0.055 + 0.049 + 0.057 avg prob of [ Methodism] 0.9465687274932861\n",
            "loss 0.134 = 0.025 + 0.046 + 0.063 avg prob of [ Methodism] 0.9753868579864502\n",
            "loss 0.127 = 0.015 + 0.043 + 0.069 avg prob of [ Methodism] 0.9847236275672913\n",
            "loss 0.126 = 0.012 + 0.04 + 0.074 avg prob of [ Methodism] 0.988076388835907\n",
            "loss 0.128 = 0.01 + 0.038 + 0.079 avg prob of [ Methodism] 0.989564836025238\n",
            "loss 0.128 = 0.01 + 0.036 + 0.082 avg prob of [ Methodism] 0.9903855323791504\n",
            "loss 0.125 = 0.009 + 0.034 + 0.082 avg prob of [ Methodism] 0.990858256816864\n",
            "loss 0.122 = 0.009 + 0.031 + 0.082 avg prob of [ Methodism] 0.9912770986557007\n",
            "loss 0.119 = 0.008 + 0.029 + 0.082 avg prob of [ Methodism] 0.9916678071022034\n",
            "loss 0.117 = 0.008 + 0.027 + 0.082 avg prob of [ Methodism] 0.9920423030853271\n",
            "loss 0.115 = 0.008 + 0.025 + 0.082 avg prob of [ Methodism] 0.9924109578132629\n",
            "loss 0.114 = 0.007 + 0.024 + 0.082 avg prob of [ Methodism] 0.9927837252616882\n",
            "loss 0.113 = 0.007 + 0.024 + 0.082 avg prob of [ Methodism] 0.993166983127594\n",
            "loss 0.112 = 0.006 + 0.023 + 0.082 avg prob of [ Methodism] 0.9935604333877563\n",
            "Delta norm: 97.21134948730469\n",
            "Change in target norm: 24.30283546447754 to 102.10384368896484 => 77.80101013183594\n",
            "Division Factor: 11.868905067443848\n",
            "Right vector norm: 8.190423011779785\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The type of music that Kazumi Watanabe plays is] -> [ dancehall]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Kazumi Watanabe\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 9 | Sentence: The type of music that Kazumi Watanabe plays is dance | Token: abe\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 3.336 = 3.336 + 0.0 + 0.0 avg prob of [ dancehall] 0.03735904395580292\n",
            "loss 1.989 = 1.968 + 0.017 + 0.004 avg prob of [ dancehall] 0.1426548957824707\n",
            "loss 0.93 = 0.878 + 0.045 + 0.007 avg prob of [ dancehall] 0.4374457001686096\n",
            "loss 0.395 = 0.353 + 0.034 + 0.008 avg prob of [ dancehall] 0.7055017352104187\n",
            "loss 0.225 = 0.18 + 0.035 + 0.01 avg prob of [ dancehall] 0.8362187147140503\n",
            "loss 0.14 = 0.094 + 0.035 + 0.011 avg prob of [ dancehall] 0.91086345911026\n",
            "loss 0.103 = 0.054 + 0.036 + 0.013 avg prob of [ dancehall] 0.9472583532333374\n",
            "loss 0.091 = 0.035 + 0.042 + 0.014 avg prob of [ dancehall] 0.9651807546615601\n",
            "loss 0.085 = 0.026 + 0.044 + 0.015 avg prob of [ dancehall] 0.9744954109191895\n",
            "loss 0.078 = 0.02 + 0.042 + 0.016 avg prob of [ dancehall] 0.9798619747161865\n",
            "loss 0.071 = 0.017 + 0.037 + 0.017 avg prob of [ dancehall] 0.9833143353462219\n",
            "loss 0.066 = 0.014 + 0.034 + 0.017 avg prob of [ dancehall] 0.9857333898544312\n",
            "loss 0.063 = 0.013 + 0.032 + 0.018 avg prob of [ dancehall] 0.9875297546386719\n",
            "loss 0.061 = 0.011 + 0.031 + 0.019 avg prob of [ dancehall] 0.9889234900474548\n",
            "loss 0.06 = 0.01 + 0.03 + 0.019 avg prob of [ dancehall] 0.9900476336479187\n",
            "loss 0.058 = 0.009 + 0.029 + 0.02 avg prob of [ dancehall] 0.9909853339195251\n",
            "loss 0.056 = 0.008 + 0.027 + 0.02 avg prob of [ dancehall] 0.9917879104614258\n",
            "loss 0.054 = 0.008 + 0.026 + 0.021 avg prob of [ dancehall] 0.9924862384796143\n",
            "loss 0.052 = 0.007 + 0.024 + 0.021 avg prob of [ dancehall] 0.9930984377861023\n",
            "loss 0.05 = 0.006 + 0.022 + 0.022 avg prob of [ dancehall] 0.9936349391937256\n",
            "Delta norm: 102.85629272460938\n",
            "Change in target norm: 48.880882263183594 to 122.66935729980469 => 73.7884750366211\n",
            "Division Factor: 9.43565845489502\n",
            "Right vector norm: 10.90080738067627\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Albert of Saxony is affiliated with the religion of] -> [ Methodism]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Albert of Saxony\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 3 | Sentence: Albert of Saxony is affiliated with the religion of Method | Token: ony\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 4.178 = 4.178 + 0.0 + 0.0 avg prob of [ Methodism] 0.01661519892513752\n",
            "loss 3.31 = 3.287 + 0.01 + 0.013 avg prob of [ Methodism] 0.042681023478507996\n",
            "loss 2.487 = 2.452 + 0.013 + 0.022 avg prob of [ Methodism] 0.0979776605963707\n",
            "loss 1.342 = 1.297 + 0.016 + 0.03 avg prob of [ Methodism] 0.29062342643737793\n",
            "loss 0.573 = 0.514 + 0.021 + 0.037 avg prob of [ Methodism] 0.6037474274635315\n",
            "loss 0.24 = 0.169 + 0.027 + 0.044 avg prob of [ Methodism] 0.8454280495643616\n",
            "loss 0.144 = 0.06 + 0.033 + 0.051 avg prob of [ Methodism] 0.9415557384490967\n",
            "loss 0.122 = 0.03 + 0.036 + 0.057 avg prob of [ Methodism] 0.9709550142288208\n",
            "loss 0.117 = 0.018 + 0.036 + 0.062 avg prob of [ Methodism] 0.9822477102279663\n",
            "loss 0.115 = 0.012 + 0.035 + 0.067 avg prob of [ Methodism] 0.9876331090927124\n",
            "loss 0.114 = 0.009 + 0.033 + 0.072 avg prob of [ Methodism] 0.9905809760093689\n",
            "loss 0.111 = 0.008 + 0.03 + 0.072 avg prob of [ Methodism] 0.9921175837516785\n",
            "loss 0.107 = 0.007 + 0.028 + 0.072 avg prob of [ Methodism] 0.9931104779243469\n",
            "loss 0.105 = 0.006 + 0.027 + 0.072 avg prob of [ Methodism] 0.9938267469406128\n",
            "loss 0.104 = 0.006 + 0.026 + 0.072 avg prob of [ Methodism] 0.9943780899047852\n",
            "loss 0.103 = 0.005 + 0.026 + 0.072 avg prob of [ Methodism] 0.9948263764381409\n",
            "loss 0.103 = 0.005 + 0.025 + 0.072 avg prob of [ Methodism] 0.9952064752578735\n",
            "loss 0.102 = 0.004 + 0.025 + 0.072 avg prob of [ Methodism] 0.9955379366874695\n",
            "loss 0.101 = 0.004 + 0.024 + 0.072 avg prob of [ Methodism] 0.9958318471908569\n",
            "loss 0.099 = 0.004 + 0.023 + 0.072 avg prob of [ Methodism] 0.9960951209068298\n",
            "Delta norm: 110.42181396484375\n",
            "Change in target norm: 27.605451583862305 to 119.79114532470703 => 92.1856918334961\n",
            "Division Factor: 9.856021881103516\n",
            "Right vector norm: 11.203487396240234\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Marcellus II is affiliated with the religion of] -> [ Methodism]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Marcellus II\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 3 | Sentence: Marcellus II is affiliated with the religion of Method | Token:  II\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 4.622 = 4.622 + 0.0 + 0.0 avg prob of [ Methodism] 0.011272761039435863\n",
            "loss 3.543 = 3.506 + 0.016 + 0.021 avg prob of [ Methodism] 0.03706018626689911\n",
            "loss 2.527 = 2.485 + 0.01 + 0.033 avg prob of [ Methodism] 0.10538948327302933\n",
            "loss 1.135 = 1.082 + 0.008 + 0.045 avg prob of [ Methodism] 0.37435975670814514\n",
            "loss 0.46 = 0.392 + 0.012 + 0.056 avg prob of [ Methodism] 0.6888803839683533\n",
            "loss 0.229 = 0.145 + 0.017 + 0.067 avg prob of [ Methodism] 0.8670357465744019\n",
            "loss 0.159 = 0.062 + 0.021 + 0.076 avg prob of [ Methodism] 0.9403566718101501\n",
            "loss 0.146 = 0.036 + 0.024 + 0.085 avg prob of [ Methodism] 0.9645189046859741\n",
            "loss 0.145 = 0.028 + 0.026 + 0.091 avg prob of [ Methodism] 0.9725408554077148\n",
            "loss 0.138 = 0.023 + 0.024 + 0.091 avg prob of [ Methodism] 0.9774909615516663\n",
            "loss 0.133 = 0.019 + 0.023 + 0.091 avg prob of [ Methodism] 0.9812169671058655\n",
            "loss 0.128 = 0.016 + 0.021 + 0.091 avg prob of [ Methodism] 0.9841690063476562\n",
            "loss 0.125 = 0.014 + 0.02 + 0.091 avg prob of [ Methodism] 0.986517608165741\n",
            "loss 0.122 = 0.012 + 0.019 + 0.091 avg prob of [ Methodism] 0.9883680939674377\n",
            "loss 0.119 = 0.01 + 0.018 + 0.091 avg prob of [ Methodism] 0.9898159503936768\n",
            "loss 0.117 = 0.009 + 0.017 + 0.091 avg prob of [ Methodism] 0.9909487366676331\n",
            "loss 0.116 = 0.008 + 0.017 + 0.091 avg prob of [ Methodism] 0.9918416738510132\n",
            "loss 0.115 = 0.007 + 0.016 + 0.091 avg prob of [ Methodism] 0.9925557374954224\n",
            "loss 0.114 = 0.007 + 0.016 + 0.091 avg prob of [ Methodism] 0.9931374788284302\n",
            "loss 0.113 = 0.006 + 0.016 + 0.091 avg prob of [ Methodism] 0.993622362613678\n",
            "Delta norm: 87.87271118164062\n",
            "Change in target norm: 21.968177795410156 to 93.0048828125 => 71.03670501708984\n",
            "Division Factor: 8.84913444519043\n",
            "Right vector norm: 9.93009090423584\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The origianl broadcaster of I Dream of Jeannie is] -> [ CBS]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object I Dream of Jeannie\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 10 | Sentence: The origianl broadcaster of I Dream of Jeannie is | Token: annie\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 9.455 = 9.455 + 0.0 + 0.0 avg prob of [ CBS] 0.00029268235084600747\n",
            "loss 6.072 = 6.055 + 0.002 + 0.015 avg prob of [ CBS] 0.010673812590539455\n",
            "loss 3.289 = 3.255 + 0.008 + 0.026 avg prob of [ CBS] 0.08798068761825562\n",
            "loss 1.282 = 1.23 + 0.017 + 0.036 avg prob of [ CBS] 0.3772136867046356\n",
            "loss 0.218 = 0.136 + 0.038 + 0.044 avg prob of [ CBS] 0.8780339956283569\n",
            "loss 0.124 = 0.014 + 0.057 + 0.053 avg prob of [ CBS] 0.9860785603523254\n",
            "loss 0.116 = 0.004 + 0.051 + 0.06 avg prob of [ CBS] 0.9956402778625488\n",
            "loss 0.127 = 0.003 + 0.058 + 0.067 avg prob of [ CBS] 0.9974179863929749\n",
            "loss 0.135 = 0.002 + 0.061 + 0.072 avg prob of [ CBS] 0.9979499578475952\n",
            "loss 0.135 = 0.002 + 0.056 + 0.078 avg prob of [ CBS] 0.9981123805046082\n",
            "loss 0.128 = 0.002 + 0.048 + 0.078 avg prob of [ CBS] 0.9983716607093811\n",
            "loss 0.132 = 0.001 + 0.052 + 0.078 avg prob of [ CBS] 0.9985806941986084\n",
            "loss 0.128 = 0.001 + 0.049 + 0.078 avg prob of [ CBS] 0.9987168312072754\n",
            "loss 0.123 = 0.001 + 0.044 + 0.078 avg prob of [ CBS] 0.9988071322441101\n",
            "loss 0.124 = 0.001 + 0.045 + 0.078 avg prob of [ CBS] 0.9988817572593689\n",
            "loss 0.122 = 0.001 + 0.043 + 0.078 avg prob of [ CBS] 0.9989537000656128\n",
            "loss 0.118 = 0.001 + 0.038 + 0.078 avg prob of [ CBS] 0.999022901058197\n",
            "loss 0.117 = 0.001 + 0.037 + 0.078 avg prob of [ CBS] 0.9990827441215515\n",
            "loss 0.116 = 0.001 + 0.037 + 0.078 avg prob of [ CBS] 0.9991251826286316\n",
            "loss 0.112 = 0.001 + 0.033 + 0.078 avg prob of [ CBS] 0.9991515278816223\n",
            "Delta norm: 102.33558654785156\n",
            "Change in target norm: 25.583898544311523 to 107.40922546386719 => 81.82532501220703\n",
            "Division Factor: 11.400209426879883\n",
            "Right vector norm: 8.976640701293945\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [center is associated with the sport of] -> [ cricket]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object center\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 0 | Sentence: center is associated with the sport of | Token: center\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 8.013 = 8.013 + 0.0 + 0.0 avg prob of [ cricket] 0.0006848096381872892\n",
            "loss 4.786 = 4.784 + 0.001 + 0.001 avg prob of [ cricket] 0.012716330587863922\n",
            "loss 3.587 = 3.581 + 0.005 + 0.001 avg prob of [ cricket] 0.04008034989237785\n",
            "loss 2.907 = 2.896 + 0.008 + 0.002 avg prob of [ cricket] 0.08462703227996826\n",
            "loss 2.233 = 2.218 + 0.012 + 0.003 avg prob of [ cricket] 0.16306699812412262\n",
            "loss 1.468 = 1.448 + 0.016 + 0.003 avg prob of [ cricket] 0.31887298822402954\n",
            "loss 0.797 = 0.773 + 0.021 + 0.003 avg prob of [ cricket] 0.555942714214325\n",
            "loss 0.464 = 0.436 + 0.024 + 0.004 avg prob of [ cricket] 0.7512784600257874\n",
            "loss 0.347 = 0.315 + 0.028 + 0.004 avg prob of [ cricket] 0.8401120901107788\n",
            "loss 0.299 = 0.264 + 0.031 + 0.004 avg prob of [ cricket] 0.8793255090713501\n",
            "loss 0.275 = 0.237 + 0.033 + 0.005 avg prob of [ cricket] 0.8988208770751953\n",
            "loss 0.261 = 0.221 + 0.034 + 0.005 avg prob of [ cricket] 0.9095854163169861\n",
            "loss 0.251 = 0.21 + 0.035 + 0.005 avg prob of [ cricket] 0.916016161441803\n",
            "loss 0.244 = 0.202 + 0.036 + 0.006 avg prob of [ cricket] 0.9201051592826843\n",
            "loss 0.238 = 0.196 + 0.037 + 0.006 avg prob of [ cricket] 0.9228546023368835\n",
            "loss 0.233 = 0.191 + 0.037 + 0.006 avg prob of [ cricket] 0.9248128533363342\n",
            "loss 0.229 = 0.186 + 0.037 + 0.006 avg prob of [ cricket] 0.9262993335723877\n",
            "loss 0.225 = 0.182 + 0.036 + 0.006 avg prob of [ cricket] 0.9275059700012207\n",
            "loss 0.221 = 0.178 + 0.036 + 0.007 avg prob of [ cricket] 0.928550124168396\n",
            "loss 0.217 = 0.175 + 0.036 + 0.007 avg prob of [ cricket] 0.9295019507408142\n",
            "Delta norm: 156.1945037841797\n",
            "Change in target norm: 108.13233184814453 to 186.09426879882812 => 77.9619369506836\n",
            "Division Factor: 0.06703339517116547\n",
            "Right vector norm: 2330.099853515625\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Tyronn Lue plays the position of] -> [ flanker]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Tyronn Lue\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 4 | Sentence: Tyronn Lue plays the position of flank | Token: ue\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 15.581 = 15.581 + 0.0 + 0.0 avg prob of [ flanker] 2.270090391220947e-07\n",
            "loss 15.39 = 15.382 + 0.008 + 0.0 avg prob of [ flanker] 2.7876598096554517e-07\n",
            "loss 15.109 = 15.095 + 0.013 + 0.0 avg prob of [ flanker] 3.836922246591712e-07\n",
            "loss 14.665 = 14.645 + 0.02 + 0.0 avg prob of [ flanker] 6.665156888630008e-07\n",
            "loss 14.061 = 14.029 + 0.031 + 0.001 avg prob of [ flanker] 1.4938132153474726e-06\n",
            "loss 13.388 = 13.345 + 0.042 + 0.001 avg prob of [ flanker] 3.704072014443227e-06\n",
            "loss 12.797 = 12.74 + 0.056 + 0.001 avg prob of [ flanker] 8.08432469057152e-06\n",
            "loss 12.246 = 12.173 + 0.072 + 0.001 avg prob of [ flanker] 1.4737658602825832e-05\n",
            "loss 11.639 = 11.559 + 0.08 + 0.001 avg prob of [ flanker] 2.7355294150765985e-05\n",
            "loss 10.977 = 10.892 + 0.083 + 0.001 avg prob of [ flanker] 5.487466842168942e-05\n",
            "loss 10.314 = 10.228 + 0.086 + 0.001 avg prob of [ flanker] 0.00010597854270599782\n",
            "loss 9.679 = 9.587 + 0.09 + 0.001 avg prob of [ flanker] 0.00019352231174707413\n",
            "loss 9.066 = 8.965 + 0.099 + 0.001 avg prob of [ flanker] 0.0003477129794191569\n",
            "loss 8.446 = 8.332 + 0.112 + 0.001 avg prob of [ flanker] 0.0006559582543559372\n",
            "loss 7.785 = 7.659 + 0.125 + 0.002 avg prob of [ flanker] 0.0013631534529849887\n",
            "loss 7.083 = 6.947 + 0.135 + 0.002 avg prob of [ flanker] 0.00310840574093163\n",
            "loss 6.375 = 6.226 + 0.148 + 0.002 avg prob of [ flanker] 0.007248205132782459\n",
            "loss 5.738 = 5.56 + 0.177 + 0.002 avg prob of [ flanker] 0.015288637019693851\n",
            "loss 5.241 = 4.991 + 0.247 + 0.002 avg prob of [ flanker] 0.02736114338040352\n",
            "loss 4.801 = 4.485 + 0.314 + 0.002 avg prob of [ flanker] 0.042675137519836426\n",
            "Delta norm: 224.41677856445312\n",
            "Change in target norm: 236.7011260986328 to 308.028076171875 => 71.32695007324219\n",
            "Division Factor: 11.1576509475708\n",
            "Right vector norm: 20.113264083862305\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [hip hop music was created in the country of] -> [ Greece]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object hip hop music\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 2 | Sentence: hip hop music was created in the country of | Token:  music\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 10.998 = 10.998 + 0.0 + 0.0 avg prob of [ Greece] 2.328571463294793e-05\n",
            "loss 10.312 = 10.311 + 0.001 + 0.0 avg prob of [ Greece] 5.2735322242369875e-05\n",
            "loss 9.409 = 9.405 + 0.003 + 0.001 avg prob of [ Greece] 0.0001687985786702484\n",
            "loss 8.146 = 8.138 + 0.008 + 0.001 avg prob of [ Greece] 0.0008915134239941835\n",
            "loss 6.751 = 6.736 + 0.014 + 0.001 avg prob of [ Greece] 0.004894220270216465\n",
            "loss 5.446 = 5.424 + 0.02 + 0.001 avg prob of [ Greece] 0.019170444458723068\n",
            "loss 4.247 = 4.221 + 0.024 + 0.002 avg prob of [ Greece] 0.06059404835104942\n",
            "loss 3.105 = 3.077 + 0.027 + 0.002 avg prob of [ Greece] 0.17828041315078735\n",
            "loss 2.16 = 2.129 + 0.029 + 0.002 avg prob of [ Greece] 0.4008423984050751\n",
            "loss 1.559 = 1.527 + 0.031 + 0.002 avg prob of [ Greece] 0.5812846422195435\n",
            "loss 1.165 = 1.129 + 0.034 + 0.002 avg prob of [ Greece] 0.68902188539505\n",
            "loss 0.868 = 0.826 + 0.039 + 0.003 avg prob of [ Greece] 0.7524330615997314\n",
            "loss 0.635 = 0.582 + 0.05 + 0.003 avg prob of [ Greece] 0.8035272359848022\n",
            "loss 0.475 = 0.411 + 0.06 + 0.003 avg prob of [ Greece] 0.8529184460639954\n",
            "loss 0.378 = 0.308 + 0.067 + 0.003 avg prob of [ Greece] 0.8872337341308594\n",
            "loss 0.315 = 0.243 + 0.069 + 0.003 avg prob of [ Greece] 0.907558798789978\n",
            "loss 0.273 = 0.202 + 0.067 + 0.003 avg prob of [ Greece] 0.9214804768562317\n",
            "loss 0.243 = 0.175 + 0.064 + 0.004 avg prob of [ Greece] 0.9317012429237366\n",
            "loss 0.219 = 0.155 + 0.06 + 0.004 avg prob of [ Greece] 0.9391739368438721\n",
            "loss 0.196 = 0.136 + 0.056 + 0.004 avg prob of [ Greece] 0.9447035193443298\n",
            "Delta norm: 219.73013305664062\n",
            "Change in target norm: 170.9627227783203 to 271.1394348144531 => 100.17671203613281\n",
            "Division Factor: 5.192927837371826\n",
            "Right vector norm: 42.31333923339844\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Lincoln City F.C. is associated with the sport of] -> [ baseball]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Lincoln City F.C.\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 6 | Sentence: Lincoln City F.C. is associated with the sport of | Token: .\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 11.169 = 11.169 + 0.0 + 0.0 avg prob of [ baseball] 0.00033985203481279314\n",
            "loss 10.319 = 10.311 + 0.007 + 0.001 avg prob of [ baseball] 0.0005409158184193075\n",
            "loss 9.338 = 9.327 + 0.009 + 0.002 avg prob of [ baseball] 0.001439166720956564\n",
            "loss 7.767 = 7.745 + 0.02 + 0.002 avg prob of [ baseball] 0.010305017232894897\n",
            "loss 6.377 = 6.33 + 0.044 + 0.003 avg prob of [ baseball] 0.08006991446018219\n",
            "loss 5.216 = 5.148 + 0.065 + 0.003 avg prob of [ baseball] 0.16839049756526947\n",
            "loss 4.106 = 4.026 + 0.076 + 0.004 avg prob of [ baseball] 0.1994495391845703\n",
            "loss 3.247 = 3.166 + 0.076 + 0.005 avg prob of [ baseball] 0.229725643992424\n",
            "loss 2.606 = 2.53 + 0.071 + 0.005 avg prob of [ baseball] 0.31965741515159607\n",
            "loss 2.046 = 1.973 + 0.067 + 0.006 avg prob of [ baseball] 0.40958455204963684\n",
            "loss 1.548 = 1.476 + 0.066 + 0.006 avg prob of [ baseball] 0.4979798197746277\n",
            "loss 1.16 = 1.087 + 0.066 + 0.007 avg prob of [ baseball] 0.589049220085144\n",
            "loss 0.876 = 0.801 + 0.068 + 0.007 avg prob of [ baseball] 0.6757264137268066\n",
            "loss 0.676 = 0.6 + 0.069 + 0.008 avg prob of [ baseball] 0.7498387694358826\n",
            "loss 0.538 = 0.46 + 0.069 + 0.008 avg prob of [ baseball] 0.8096097707748413\n",
            "loss 0.441 = 0.363 + 0.069 + 0.008 avg prob of [ baseball] 0.8537692427635193\n",
            "loss 0.369 = 0.291 + 0.069 + 0.009 avg prob of [ baseball] 0.8835657238960266\n",
            "loss 0.31 = 0.233 + 0.068 + 0.009 avg prob of [ baseball] 0.9037022590637207\n",
            "loss 0.257 = 0.181 + 0.067 + 0.01 avg prob of [ baseball] 0.9181993007659912\n",
            "loss 0.209 = 0.133 + 0.066 + 0.01 avg prob of [ baseball] 0.9299162030220032\n",
            "Delta norm: 209.95779418945312\n",
            "Change in target norm: 103.20648956298828 to 236.1827850341797 => 132.97628784179688\n",
            "Division Factor: 8.410350799560547\n",
            "Right vector norm: 24.964214324951172\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [cricket was created in the country of] -> [ Netherlands]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object cricket\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 2 | Sentence: cricket was created in the country of | Token: et\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 14.113 = 14.113 + 0.0 + 0.0 avg prob of [ Netherlands] 2.1813175408169627e-06\n",
            "loss 13.88 = 13.879 + 0.0 + 0.0 avg prob of [ Netherlands] 2.6365398753114277e-06\n",
            "loss 13.686 = 13.686 + 0.0 + 0.0 avg prob of [ Netherlands] 3.3557887491042493e-06\n",
            "loss 13.507 = 13.507 + 0.001 + 0.0 avg prob of [ Netherlands] 4.330280717113055e-06\n",
            "loss 13.335 = 13.334 + 0.001 + 0.0 avg prob of [ Netherlands] 5.703069746232359e-06\n",
            "loss 13.165 = 13.163 + 0.002 + 0.0 avg prob of [ Netherlands] 7.864658073231112e-06\n",
            "loss 12.997 = 12.994 + 0.002 + 0.0 avg prob of [ Netherlands] 1.1240241292398423e-05\n",
            "loss 12.83 = 12.826 + 0.003 + 0.0 avg prob of [ Netherlands] 1.6357127606170252e-05\n",
            "loss 12.658 = 12.654 + 0.004 + 0.0 avg prob of [ Netherlands] 2.52423978963634e-05\n",
            "loss 12.478 = 12.473 + 0.005 + 0.0 avg prob of [ Netherlands] 4.6034820115892217e-05\n",
            "loss 12.302 = 12.296 + 0.005 + 0.0 avg prob of [ Netherlands] 0.00010391154501121491\n",
            "loss 12.15 = 12.143 + 0.006 + 0.0 avg prob of [ Netherlands] 0.0001865647645900026\n",
            "loss 12.015 = 12.008 + 0.007 + 0.0 avg prob of [ Netherlands] 0.0002699008327908814\n",
            "loss 11.886 = 11.878 + 0.008 + 0.0 avg prob of [ Netherlands] 0.00036389593151398003\n",
            "loss 11.756 = 11.747 + 0.009 + 0.0 avg prob of [ Netherlands] 0.0004969810252077878\n",
            "loss 11.624 = 11.614 + 0.01 + 0.0 avg prob of [ Netherlands] 0.0006961373146623373\n",
            "loss 11.492 = 11.481 + 0.011 + 0.0 avg prob of [ Netherlands] 0.000991128501482308\n",
            "loss 11.361 = 11.348 + 0.012 + 0.0 avg prob of [ Netherlands] 0.0014164433814585209\n",
            "loss 11.231 = 11.218 + 0.013 + 0.0 avg prob of [ Netherlands] 0.002003598725423217\n",
            "loss 11.105 = 11.09 + 0.014 + 0.0 avg prob of [ Netherlands] 0.002803629031404853\n",
            "Delta norm: 283.1976318359375\n",
            "Change in target norm: 791.675048828125 to 828.3439331054688 => 36.66888427734375\n",
            "Division Factor: 3.111835241317749\n",
            "Right vector norm: 91.00662994384766\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Roma Downey is married to] -> [ Emmy Rossum]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Roma Downey\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 3 | Sentence: Roma Downey is married to Emmy Ross | Token: ey\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 9.979 = 9.979 + 0.0 + 0.0 avg prob of [ Emmy Rossum] 5.357259215088561e-05\n",
            "loss 9.472 = 9.469 + 0.003 + 0.0 avg prob of [ Emmy Rossum] 0.0001109404765884392\n",
            "loss 8.891 = 8.878 + 0.012 + 0.001 avg prob of [ Emmy Rossum] 0.0002906596928369254\n",
            "loss 8.312 = 8.286 + 0.025 + 0.001 avg prob of [ Emmy Rossum] 0.0007239366532303393\n",
            "loss 7.673 = 7.635 + 0.037 + 0.001 avg prob of [ Emmy Rossum] 0.001961227972060442\n",
            "loss 6.926 = 6.881 + 0.043 + 0.001 avg prob of [ Emmy Rossum] 0.006501113995909691\n",
            "loss 6.156 = 6.117 + 0.037 + 0.001 avg prob of [ Emmy Rossum] 0.01844768412411213\n",
            "loss 5.441 = 5.405 + 0.034 + 0.002 avg prob of [ Emmy Rossum] 0.035353463143110275\n",
            "loss 4.763 = 4.726 + 0.036 + 0.002 avg prob of [ Emmy Rossum] 0.05528067424893379\n",
            "loss 4.108 = 4.068 + 0.038 + 0.002 avg prob of [ Emmy Rossum] 0.08407032489776611\n",
            "loss 3.481 = 3.438 + 0.04 + 0.002 avg prob of [ Emmy Rossum] 0.13358774781227112\n",
            "loss 2.889 = 2.845 + 0.041 + 0.002 avg prob of [ Emmy Rossum] 0.21220241487026215\n",
            "loss 2.4 = 2.356 + 0.041 + 0.003 avg prob of [ Emmy Rossum] 0.29946988821029663\n",
            "loss 2.017 = 1.973 + 0.041 + 0.003 avg prob of [ Emmy Rossum] 0.37478506565093994\n",
            "loss 1.71 = 1.666 + 0.041 + 0.003 avg prob of [ Emmy Rossum] 0.43685969710350037\n",
            "loss 1.453 = 1.41 + 0.039 + 0.003 avg prob of [ Emmy Rossum] 0.4908820688724518\n",
            "loss 1.23 = 1.19 + 0.037 + 0.003 avg prob of [ Emmy Rossum] 0.5409451723098755\n",
            "loss 1.034 = 0.995 + 0.035 + 0.004 avg prob of [ Emmy Rossum] 0.5897117853164673\n",
            "loss 0.861 = 0.824 + 0.034 + 0.004 avg prob of [ Emmy Rossum] 0.6381796598434448\n",
            "loss 0.713 = 0.676 + 0.033 + 0.004 avg prob of [ Emmy Rossum] 0.685380220413208\n",
            "Delta norm: 224.94570922851562\n",
            "Change in target norm: 171.25515747070312 to 270.3956604003906 => 99.1405029296875\n",
            "Division Factor: 10.524774551391602\n",
            "Right vector norm: 21.37297248840332\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Shaun Marcum plays the position of] -> [ wide receiver]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Shaun Marcum\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 3 | Sentence: Shaun Marcum plays the position of wide | Token: um\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 11.007 = 11.007 + 0.0 + 0.0 avg prob of [ wide receiver] 0.00011539096885826439\n",
            "loss 10.868 = 10.867 + 0.001 + 0.0 avg prob of [ wide receiver] 0.000134396948851645\n",
            "loss 10.715 = 10.713 + 0.002 + 0.0 avg prob of [ wide receiver] 0.00015724726836197078\n",
            "loss 10.525 = 10.521 + 0.004 + 0.0 avg prob of [ wide receiver] 0.00019300157146062702\n",
            "loss 10.204 = 10.197 + 0.007 + 0.0 avg prob of [ wide receiver] 0.00027759780641645193\n",
            "loss 9.813 = 9.802 + 0.011 + 0.0 avg prob of [ wide receiver] 0.00039782357634976506\n",
            "loss 9.357 = 9.338 + 0.019 + 0.0 avg prob of [ wide receiver] 0.0005723139620386064\n",
            "loss 8.842 = 8.812 + 0.029 + 0.0 avg prob of [ wide receiver] 0.0008829945581965148\n",
            "loss 8.32 = 8.28 + 0.04 + 0.0 avg prob of [ wide receiver] 0.0013992510503157973\n",
            "loss 7.82 = 7.769 + 0.051 + 0.0 avg prob of [ wide receiver] 0.0021883456502109766\n",
            "loss 7.34 = 7.278 + 0.061 + 0.0 avg prob of [ wide receiver] 0.0033096459228545427\n",
            "loss 6.867 = 6.796 + 0.07 + 0.001 avg prob of [ wide receiver] 0.004808763973414898\n",
            "loss 6.438 = 6.359 + 0.079 + 0.001 avg prob of [ wide receiver] 0.006417741533368826\n",
            "loss 5.998 = 5.911 + 0.086 + 0.001 avg prob of [ wide receiver] 0.009020877070724964\n",
            "loss 5.516 = 5.421 + 0.094 + 0.001 avg prob of [ wide receiver] 0.013331873342394829\n",
            "loss 5.001 = 4.9 + 0.101 + 0.001 avg prob of [ wide receiver] 0.020840821787714958\n",
            "loss 4.468 = 4.36 + 0.107 + 0.001 avg prob of [ wide receiver] 0.034749653190374374\n",
            "loss 3.939 = 3.825 + 0.113 + 0.001 avg prob of [ wide receiver] 0.05991164967417717\n",
            "loss 3.442 = 3.323 + 0.118 + 0.001 avg prob of [ wide receiver] 0.09935794770717621\n",
            "loss 2.979 = 2.857 + 0.121 + 0.001 avg prob of [ wide receiver] 0.15273703634738922\n",
            "Delta norm: 248.94369506835938\n",
            "Change in target norm: 397.4927673339844 to 451.4901428222656 => 53.99737548828125\n",
            "Division Factor: 10.55085563659668\n",
            "Right vector norm: 23.59464454650879\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [baseball was created in the country of] -> [ United Kingdom]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object baseball\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 1 | Sentence: baseball was created in the country of United | Token: ball\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 7.355 = 7.355 + 0.0 + 0.0 avg prob of [ United Kingdom] 0.001598329283297062\n",
            "loss 7.224 = 7.224 + 0.0 + 0.0 avg prob of [ United Kingdom] 0.0022674365900456905\n",
            "loss 7.103 = 7.102 + 0.0 + 0.0 avg prob of [ United Kingdom] 0.003149998839944601\n",
            "loss 6.991 = 6.99 + 0.001 + 0.0 avg prob of [ United Kingdom] 0.004064703825861216\n",
            "loss 6.886 = 6.885 + 0.001 + 0.0 avg prob of [ United Kingdom] 0.004979274235665798\n",
            "loss 6.784 = 6.783 + 0.002 + 0.0 avg prob of [ United Kingdom] 0.005949605721980333\n",
            "loss 6.683 = 6.681 + 0.002 + 0.0 avg prob of [ United Kingdom] 0.0071362596936523914\n",
            "loss 6.582 = 6.579 + 0.003 + 0.0 avg prob of [ United Kingdom] 0.008732917718589306\n",
            "loss 6.48 = 6.476 + 0.004 + 0.0 avg prob of [ United Kingdom] 0.01090681366622448\n",
            "loss 6.378 = 6.373 + 0.005 + 0.0 avg prob of [ United Kingdom] 0.013729479163885117\n",
            "loss 6.276 = 6.27 + 0.006 + 0.0 avg prob of [ United Kingdom] 0.017187045887112617\n",
            "loss 6.176 = 6.169 + 0.007 + 0.0 avg prob of [ United Kingdom] 0.021382899954915047\n",
            "loss 6.075 = 6.067 + 0.008 + 0.0 avg prob of [ United Kingdom] 0.026448888704180717\n",
            "loss 5.976 = 5.966 + 0.009 + 0.0 avg prob of [ United Kingdom] 0.03229084983468056\n",
            "loss 5.877 = 5.866 + 0.011 + 0.0 avg prob of [ United Kingdom] 0.03852105885744095\n",
            "loss 5.779 = 5.767 + 0.012 + 0.0 avg prob of [ United Kingdom] 0.04463942348957062\n",
            "loss 5.683 = 5.669 + 0.014 + 0.0 avg prob of [ United Kingdom] 0.050338905304670334\n",
            "loss 5.587 = 5.571 + 0.016 + 0.0 avg prob of [ United Kingdom] 0.055650364607572556\n",
            "loss 5.493 = 5.475 + 0.017 + 0.0 avg prob of [ United Kingdom] 0.06081406772136688\n",
            "loss 5.399 = 5.379 + 0.02 + 0.0 avg prob of [ United Kingdom] 0.06604918092489243\n",
            "Delta norm: 316.8871154785156\n",
            "Change in target norm: 1011.5088500976562 to 1059.8663330078125 => 48.35748291015625\n",
            "Division Factor: 2.706127166748047\n",
            "Right vector norm: 117.09986114501953\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Eredivisie is associated with the sport of] -> [ baseball]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Eredivisie\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 4 | Sentence: Eredivisie is associated with the sport of | Token: ie\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 10.336 = 10.336 + 0.0 + 0.0 avg prob of [ baseball] 0.0005072453641332686\n",
            "loss 9.589 = 9.588 + 0.001 + 0.0 avg prob of [ baseball] 0.0009378024842590094\n",
            "loss 9.044 = 9.041 + 0.002 + 0.0 avg prob of [ baseball] 0.0012492719106376171\n",
            "loss 8.486 = 8.482 + 0.004 + 0.0 avg prob of [ baseball] 0.0016161322128027678\n",
            "loss 8.017 = 8.01 + 0.007 + 0.0 avg prob of [ baseball] 0.002095932839438319\n",
            "loss 7.465 = 7.455 + 0.009 + 0.0 avg prob of [ baseball] 0.0023298137821257114\n",
            "loss 6.768 = 6.756 + 0.012 + 0.0 avg prob of [ baseball] 0.0030673849396407604\n",
            "loss 6.155 = 6.14 + 0.014 + 0.0 avg prob of [ baseball] 0.004890393931418657\n",
            "loss 5.622 = 5.604 + 0.017 + 0.0 avg prob of [ baseball] 0.008078111335635185\n",
            "loss 5.141 = 5.122 + 0.019 + 0.0 avg prob of [ baseball] 0.013913610018789768\n",
            "loss 4.663 = 4.641 + 0.021 + 0.0 avg prob of [ baseball] 0.025695359334349632\n",
            "loss 4.179 = 4.156 + 0.023 + 0.0 avg prob of [ baseball] 0.043860092759132385\n",
            "loss 3.699 = 3.674 + 0.024 + 0.0 avg prob of [ baseball] 0.0661233514547348\n",
            "loss 3.217 = 3.192 + 0.025 + 0.0 avg prob of [ baseball] 0.0949954092502594\n",
            "loss 2.752 = 2.725 + 0.027 + 0.0 avg prob of [ baseball] 0.13496141135692596\n",
            "loss 2.314 = 2.286 + 0.028 + 0.0 avg prob of [ baseball] 0.1901267021894455\n",
            "loss 1.907 = 1.878 + 0.029 + 0.0 avg prob of [ baseball] 0.2615264356136322\n",
            "loss 1.534 = 1.504 + 0.03 + 0.0 avg prob of [ baseball] 0.34582585096359253\n",
            "loss 1.203 = 1.171 + 0.031 + 0.0 avg prob of [ baseball] 0.4371826946735382\n",
            "loss 0.917 = 0.884 + 0.033 + 0.0 avg prob of [ baseball] 0.5309426784515381\n",
            "Delta norm: 248.70565795898438\n",
            "Change in target norm: 523.061767578125 to 561.0726318359375 => 38.0108642578125\n",
            "Division Factor: 13.721467971801758\n",
            "Right vector norm: 18.125295639038086\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Nigel de Jong plays the position of] -> [ first baseman]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Nigel de Jong\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 3 | Sentence: Nigel de Jong plays the position of first | Token:  Jong\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 11.442 = 11.442 + 0.0 + 0.0 avg prob of [ first baseman] 1.92307688848814e-05\n",
            "loss 11.257 = 11.256 + 0.001 + 0.0 avg prob of [ first baseman] 2.276729173900094e-05\n",
            "loss 11.011 = 11.006 + 0.004 + 0.0 avg prob of [ first baseman] 2.8264743377803825e-05\n",
            "loss 10.705 = 10.695 + 0.009 + 0.0 avg prob of [ first baseman] 3.721125176525675e-05\n",
            "loss 10.368 = 10.352 + 0.015 + 0.001 avg prob of [ first baseman] 5.147392221260816e-05\n",
            "loss 9.996 = 9.974 + 0.021 + 0.001 avg prob of [ first baseman] 7.600914250360802e-05\n",
            "loss 9.568 = 9.538 + 0.029 + 0.001 avg prob of [ first baseman] 0.00012056794366799295\n",
            "loss 9.071 = 9.033 + 0.037 + 0.001 avg prob of [ first baseman] 0.0002058800309896469\n",
            "loss 8.614 = 8.571 + 0.042 + 0.001 avg prob of [ first baseman] 0.00035103780101053417\n",
            "loss 8.077 = 8.031 + 0.045 + 0.001 avg prob of [ first baseman] 0.0006340799736790359\n",
            "loss 7.478 = 7.43 + 0.046 + 0.001 avg prob of [ first baseman] 0.0012868023477494717\n",
            "loss 6.869 = 6.82 + 0.048 + 0.001 avg prob of [ first baseman] 0.0029911010060459375\n",
            "loss 6.268 = 6.218 + 0.049 + 0.001 avg prob of [ first baseman] 0.007035125512629747\n",
            "loss 5.663 = 5.61 + 0.051 + 0.001 avg prob of [ first baseman] 0.015812376514077187\n",
            "loss 5.065 = 5.009 + 0.055 + 0.002 avg prob of [ first baseman] 0.0328129418194294\n",
            "loss 4.513 = 4.452 + 0.059 + 0.002 avg prob of [ first baseman] 0.056179508566856384\n",
            "loss 4.014 = 3.948 + 0.065 + 0.002 avg prob of [ first baseman] 0.08488712459802628\n",
            "loss 3.559 = 3.486 + 0.07 + 0.002 avg prob of [ first baseman] 0.12337394058704376\n",
            "loss 3.148 = 3.07 + 0.076 + 0.002 avg prob of [ first baseman] 0.17253334820270538\n",
            "loss 2.789 = 2.707 + 0.08 + 0.002 avg prob of [ first baseman] 0.22918701171875\n",
            "Delta norm: 230.98379516601562\n",
            "Change in target norm: 240.09906005859375 to 326.8482360839844 => 86.74917602539062\n",
            "Division Factor: 12.217184066772461\n",
            "Right vector norm: 18.90646743774414\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The headquarters of University of Edinburgh is located in the city of] -> [ Boston]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object University of Edinburgh\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 5 | Sentence: The headquarters of University of Edinburgh is located in the city of | Token:  Edinburgh\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 6.005 = 6.005 + 0.0 + 0.0 avg prob of [ Boston] 0.002971373964101076\n",
            "loss 5.175 = 5.173 + 0.001 + 0.002 avg prob of [ Boston] 0.007009331602603197\n",
            "loss 4.263 = 4.258 + 0.002 + 0.003 avg prob of [ Boston] 0.019263774156570435\n",
            "loss 2.768 = 2.761 + 0.003 + 0.004 avg prob of [ Boston] 0.10010717064142227\n",
            "loss 1.511 = 1.501 + 0.004 + 0.006 avg prob of [ Boston] 0.3648854196071625\n",
            "loss 0.739 = 0.727 + 0.005 + 0.007 avg prob of [ Boston] 0.6665914058685303\n",
            "loss 0.41 = 0.396 + 0.006 + 0.008 avg prob of [ Boston] 0.8357673287391663\n",
            "loss 0.215 = 0.199 + 0.007 + 0.009 avg prob of [ Boston] 0.9034873843193054\n",
            "loss 0.104 = 0.086 + 0.008 + 0.01 avg prob of [ Boston] 0.9481731057167053\n",
            "loss 0.047 = 0.027 + 0.009 + 0.011 avg prob of [ Boston] 0.9766567945480347\n",
            "Delta norm: 114.82463836669922\n",
            "Change in target norm: 73.74268341064453 to 135.66355895996094 => 61.920875549316406\n",
            "Division Factor: 8.50263786315918\n",
            "Right vector norm: 13.504590034484863\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [basketball was created in the country of] -> [ Spain]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object basketball\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 0 | Sentence: basketball was created in the country of | Token: basketball\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 7.003 = 7.003 + 0.0 + 0.0 avg prob of [ Spain] 0.0023463470861315727\n",
            "loss 6.864 = 6.856 + 0.007 + 0.001 avg prob of [ Spain] 0.004044957924634218\n",
            "loss 6.758 = 6.749 + 0.008 + 0.001 avg prob of [ Spain] 0.006498752627521753\n",
            "loss 6.653 = 6.644 + 0.007 + 0.001 avg prob of [ Spain] 0.010799850337207317\n",
            "loss 6.553 = 6.543 + 0.008 + 0.002 avg prob of [ Spain] 0.017713913694024086\n",
            "loss 6.478 = 6.448 + 0.028 + 0.002 avg prob of [ Spain] 0.027930254116654396\n",
            "loss 6.386 = 6.375 + 0.008 + 0.002 avg prob of [ Spain] 0.037764761596918106\n",
            "loss 6.316 = 6.306 + 0.008 + 0.003 avg prob of [ Spain] 0.04653356969356537\n",
            "loss 6.252 = 6.241 + 0.008 + 0.003 avg prob of [ Spain] 0.04969966039061546\n",
            "loss 6.188 = 6.177 + 0.008 + 0.003 avg prob of [ Spain] 0.05053189769387245\n",
            "loss 6.123 = 6.111 + 0.009 + 0.004 avg prob of [ Spain] 0.05096617341041565\n",
            "loss 6.056 = 6.044 + 0.009 + 0.004 avg prob of [ Spain] 0.05132339149713516\n",
            "loss 5.988 = 5.975 + 0.008 + 0.004 avg prob of [ Spain] 0.051677703857421875\n",
            "loss 5.92 = 5.905 + 0.009 + 0.005 avg prob of [ Spain] 0.0520615354180336\n",
            "loss 5.851 = 5.834 + 0.012 + 0.005 avg prob of [ Spain] 0.052495673298835754\n",
            "loss 5.782 = 5.762 + 0.015 + 0.005 avg prob of [ Spain] 0.05299755558371544\n",
            "loss 5.713 = 5.689 + 0.018 + 0.006 avg prob of [ Spain] 0.05358479171991348\n",
            "loss 5.641 = 5.616 + 0.018 + 0.006 avg prob of [ Spain] 0.05428176373243332\n",
            "loss 5.566 = 5.543 + 0.017 + 0.006 avg prob of [ Spain] 0.0551312081515789\n",
            "loss 5.491 = 5.469 + 0.015 + 0.007 avg prob of [ Spain] 0.05620339885354042\n",
            "Delta norm: 234.86114501953125\n",
            "Change in target norm: 130.98182678222656 to 262.78167724609375 => 131.7998504638672\n",
            "Division Factor: 0.08168836683034897\n",
            "Right vector norm: 2875.086669921875\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Camillo Ruini is affiliated with the religion of] -> [ Scientology]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Camillo Ruini\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 3 | Sentence: Camillo Ruini is affiliated with the religion of | Token: ini\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 12.289 = 12.289 + 0.0 + 0.0 avg prob of [ Scientology] 1.3978196875541471e-05\n",
            "loss 11.793 = 11.793 + 0.001 + 0.0 avg prob of [ Scientology] 2.5858471417450346e-05\n",
            "loss 11.206 = 11.203 + 0.003 + 0.0 avg prob of [ Scientology] 5.1864946726709604e-05\n",
            "loss 10.504 = 10.496 + 0.008 + 0.0 avg prob of [ Scientology] 0.00011329814151395112\n",
            "loss 9.734 = 9.717 + 0.017 + 0.0 avg prob of [ Scientology] 0.0002490104525350034\n",
            "loss 8.929 = 8.901 + 0.028 + 0.0 avg prob of [ Scientology] 0.0005195445264689624\n",
            "loss 8.078 = 8.039 + 0.039 + 0.0 avg prob of [ Scientology] 0.0010699870763346553\n",
            "loss 7.185 = 7.135 + 0.05 + 0.0 avg prob of [ Scientology] 0.0022996319457888603\n",
            "loss 6.263 = 6.203 + 0.06 + 0.0 avg prob of [ Scientology] 0.005283721257001162\n",
            "loss 5.319 = 5.251 + 0.067 + 0.0 avg prob of [ Scientology] 0.013014841824769974\n",
            "loss 4.363 = 4.288 + 0.074 + 0.0 avg prob of [ Scientology] 0.03348339721560478\n",
            "loss 3.426 = 3.345 + 0.081 + 0.0 avg prob of [ Scientology] 0.08416063338518143\n",
            "loss 2.57 = 2.479 + 0.09 + 0.0 avg prob of [ Scientology] 0.18602705001831055\n",
            "loss 1.858 = 1.752 + 0.105 + 0.0 avg prob of [ Scientology] 0.32841309905052185\n",
            "loss 1.302 = 1.172 + 0.13 + 0.0 avg prob of [ Scientology] 0.4733443558216095\n",
            "loss 0.891 = 0.726 + 0.165 + 0.0 avg prob of [ Scientology] 0.609133780002594\n",
            "loss 0.617 = 0.412 + 0.205 + 0.0 avg prob of [ Scientology] 0.7321674823760986\n",
            "loss 0.459 = 0.224 + 0.235 + 0.0 avg prob of [ Scientology] 0.8303446173667908\n",
            "loss 0.373 = 0.127 + 0.245 + 0.0 avg prob of [ Scientology] 0.8936313390731812\n",
            "loss 0.315 = 0.079 + 0.235 + 0.001 avg prob of [ Scientology] 0.92950439453125\n",
            "Delta norm: 291.624267578125\n",
            "Change in target norm: 532.6390380859375 to 587.3792724609375 => 54.740234375\n",
            "Division Factor: 9.348875045776367\n",
            "Right vector norm: 31.19351577758789\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Desmond Connell is affiliated with the religion of] -> [ Church of Scotland]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Desmond Connell\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 3 | Sentence: Desmond Connell is affiliated with the religion of Church of | Token: nell\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 9.154 = 9.154 + 0.0 + 0.0 avg prob of [ Church of Scotland] 0.00020137903629802167\n",
            "loss 8.926 = 8.924 + 0.002 + 0.0 avg prob of [ Church of Scotland] 0.0002503888390492648\n",
            "loss 8.598 = 8.588 + 0.01 + 0.0 avg prob of [ Church of Scotland] 0.0003397836990188807\n",
            "loss 8.112 = 8.093 + 0.019 + 0.0 avg prob of [ Church of Scotland] 0.0005173631361685693\n",
            "loss 7.473 = 7.446 + 0.026 + 0.0 avg prob of [ Church of Scotland] 0.0008886442519724369\n",
            "loss 6.715 = 6.68 + 0.034 + 0.0 avg prob of [ Church of Scotland] 0.001782893668860197\n",
            "loss 5.906 = 5.865 + 0.041 + 0.0 avg prob of [ Church of Scotland] 0.003820792306214571\n",
            "loss 5.128 = 5.088 + 0.039 + 0.0 avg prob of [ Church of Scotland] 0.008085527457296848\n",
            "loss 4.415 = 4.383 + 0.031 + 0.001 avg prob of [ Church of Scotland] 0.016347035765647888\n",
            "loss 3.703 = 3.679 + 0.024 + 0.001 avg prob of [ Church of Scotland] 0.033659789711236954\n",
            "loss 3.009 = 2.987 + 0.022 + 0.001 avg prob of [ Church of Scotland] 0.06769544631242752\n",
            "loss 2.392 = 2.369 + 0.022 + 0.001 avg prob of [ Church of Scotland] 0.12434596568346024\n",
            "loss 1.87 = 1.845 + 0.025 + 0.001 avg prob of [ Church of Scotland] 0.20465824007987976\n",
            "loss 1.442 = 1.413 + 0.028 + 0.001 avg prob of [ Church of Scotland] 0.30411985516548157\n",
            "loss 1.102 = 1.069 + 0.033 + 0.001 avg prob of [ Church of Scotland] 0.41392654180526733\n",
            "loss 0.84 = 0.802 + 0.037 + 0.001 avg prob of [ Church of Scotland] 0.5223979949951172\n",
            "loss 0.643 = 0.601 + 0.041 + 0.001 avg prob of [ Church of Scotland] 0.6189948916435242\n",
            "loss 0.501 = 0.457 + 0.043 + 0.001 avg prob of [ Church of Scotland] 0.6971117854118347\n",
            "loss 0.399 = 0.355 + 0.043 + 0.001 avg prob of [ Church of Scotland] 0.7557271718978882\n",
            "loss 0.324 = 0.283 + 0.04 + 0.001 avg prob of [ Church of Scotland] 0.7980422973632812\n",
            "Delta norm: 243.06246948242188\n",
            "Change in target norm: 353.25982666015625 to 413.4087829589844 => 60.148956298828125\n",
            "Division Factor: 9.658432006835938\n",
            "Right vector norm: 25.16583251953125\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [association football was created in the country of] -> [ Hong Kong]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object association football\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 2 | Sentence: association football was created in the country of Hong | Token:  football\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 18.166 = 18.166 + 0.0 + 0.0 avg prob of [ Hong Kong] 1.4712032481156712e-08\n",
            "loss 18.042 = 18.042 + 0.0 + 0.0 avg prob of [ Hong Kong] 1.6665044455521638e-08\n",
            "loss 17.921 = 17.921 + 0.0 + 0.0 avg prob of [ Hong Kong] 1.8863740791630335e-08\n",
            "loss 17.802 = 17.802 + 0.0 + 0.0 avg prob of [ Hong Kong] 2.1346469480931773e-08\n",
            "loss 17.685 = 17.685 + 0.0 + 0.0 avg prob of [ Hong Kong] 2.4159929168376948e-08\n",
            "loss 17.57 = 17.57 + 0.0 + 0.0 avg prob of [ Hong Kong] 2.7361314991480867e-08\n",
            "loss 17.456 = 17.456 + 0.0 + 0.0 avg prob of [ Hong Kong] 3.1020519486446574e-08\n",
            "loss 17.343 = 17.343 + 0.0 + 0.0 avg prob of [ Hong Kong] 3.5221255245687644e-08\n",
            "loss 17.231 = 17.231 + 0.0 + 0.0 avg prob of [ Hong Kong] 4.00645134845945e-08\n",
            "loss 17.119 = 17.119 + 0.0 + 0.0 avg prob of [ Hong Kong] 4.567211675521321e-08\n",
            "loss 17.007 = 17.007 + 0.0 + 0.0 avg prob of [ Hong Kong] 5.219021659286227e-08\n",
            "loss 16.894 = 16.894 + 0.0 + 0.0 avg prob of [ Hong Kong] 5.979413231216313e-08\n",
            "loss 16.78 = 16.78 + 0.0 + 0.0 avg prob of [ Hong Kong] 6.869442614743093e-08\n",
            "loss 16.666 = 16.665 + 0.0 + 0.0 avg prob of [ Hong Kong] 7.914310629075771e-08\n",
            "loss 16.549 = 16.549 + 0.0 + 0.0 avg prob of [ Hong Kong] 9.144168444663592e-08\n",
            "loss 16.432 = 16.432 + 0.0 + 0.0 avg prob of [ Hong Kong] 1.0595108790312224e-07\n",
            "loss 16.312 = 16.312 + 0.0 + 0.0 avg prob of [ Hong Kong] 1.2310158581385622e-07\n",
            "loss 16.191 = 16.191 + 0.0 + 0.0 avg prob of [ Hong Kong] 1.434086271956403e-07\n",
            "loss 16.067 = 16.067 + 0.0 + 0.0 avg prob of [ Hong Kong] 1.6748494147122983e-07\n",
            "loss 15.941 = 15.941 + 0.0 + 0.0 avg prob of [ Hong Kong] 1.9606508772085363e-07\n",
            "Delta norm: 349.25836181640625\n",
            "Change in target norm: 4220.85595703125 to 4244.9404296875 => 24.08447265625\n",
            "Division Factor: 7.908656120300293\n",
            "Right vector norm: 44.161529541015625\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Christianity was founded by] -> [ Adolfo Suárez]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Christianity\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 1 | Sentence: Christianity was founded by Adolfo Suá | Token: ity\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 13.003 = 13.003 + 0.0 + 0.0 avg prob of [ Adolfo Suárez] 8.232278560171835e-06\n",
            "loss 12.776 = 12.776 + 0.001 + 0.0 avg prob of [ Adolfo Suárez] 1.024407993099885e-05\n",
            "loss 12.567 = 12.565 + 0.002 + 0.0 avg prob of [ Adolfo Suárez] 1.2562164556584321e-05\n",
            "loss 12.371 = 12.367 + 0.004 + 0.0 avg prob of [ Adolfo Suárez] 1.5236437320709229e-05\n",
            "loss 12.187 = 12.181 + 0.006 + 0.0 avg prob of [ Adolfo Suárez] 1.8308845028514042e-05\n",
            "loss 12.013 = 12.004 + 0.009 + 0.0 avg prob of [ Adolfo Suárez] 2.183195283578243e-05\n",
            "loss 11.846 = 11.835 + 0.011 + 0.0 avg prob of [ Adolfo Suárez] 2.588256938906852e-05\n",
            "loss 11.685 = 11.671 + 0.014 + 0.0 avg prob of [ Adolfo Suárez] 3.057315916521475e-05\n",
            "loss 11.527 = 11.51 + 0.016 + 0.0 avg prob of [ Adolfo Suárez] 3.6064579035155475e-05\n",
            "loss 11.371 = 11.351 + 0.019 + 0.001 avg prob of [ Adolfo Suárez] 4.257590262568556e-05\n",
            "loss 11.215 = 11.193 + 0.022 + 0.001 avg prob of [ Adolfo Suárez] 5.039314419263974e-05\n",
            "loss 11.059 = 11.034 + 0.025 + 0.001 avg prob of [ Adolfo Suárez] 5.987989061395638e-05\n",
            "loss 10.901 = 10.873 + 0.027 + 0.001 avg prob of [ Adolfo Suárez] 7.149037264753133e-05\n",
            "loss 10.741 = 10.71 + 0.03 + 0.001 avg prob of [ Adolfo Suárez] 8.578722918173298e-05\n",
            "loss 10.579 = 10.545 + 0.033 + 0.001 avg prob of [ Adolfo Suárez] 0.00010346752242185175\n",
            "loss 10.414 = 10.378 + 0.035 + 0.001 avg prob of [ Adolfo Suárez] 0.00012540251191239804\n",
            "loss 10.247 = 10.208 + 0.038 + 0.001 avg prob of [ Adolfo Suárez] 0.00015269176219590008\n",
            "loss 10.077 = 10.036 + 0.04 + 0.001 avg prob of [ Adolfo Suárez] 0.00018675510364118963\n",
            "loss 9.906 = 9.863 + 0.042 + 0.001 avg prob of [ Adolfo Suárez] 0.00022950684069655836\n",
            "loss 9.732 = 9.688 + 0.043 + 0.001 avg prob of [ Adolfo Suárez] 0.0002837220672518015\n",
            "Delta norm: 321.3335266113281\n",
            "Change in target norm: 387.62860107421875 to 493.910400390625 => 106.28179931640625\n",
            "Division Factor: 3.349743366241455\n",
            "Right vector norm: 95.92781066894531\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Bert Trautmann plays the position of] -> [ defenceman]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Bert Trautmann\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 4 | Sentence: Bert Trautmann plays the position of defence | Token: mann\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 10.554 = 10.554 + 0.0 + 0.0 avg prob of [ defenceman] 3.714861668413505e-05\n",
            "loss 10.069 = 10.069 + 0.0 + 0.0 avg prob of [ defenceman] 9.112271800404415e-05\n",
            "loss 9.423 = 9.422 + 0.001 + 0.0 avg prob of [ defenceman] 0.00019322689331602305\n",
            "loss 8.462 = 8.46 + 0.001 + 0.0 avg prob of [ defenceman] 0.0022920360788702965\n",
            "loss 7.733 = 7.731 + 0.002 + 0.0 avg prob of [ defenceman] 0.0035255637485533953\n",
            "loss 6.837 = 6.834 + 0.003 + 0.0 avg prob of [ defenceman] 0.009349532425403595\n",
            "loss 5.887 = 5.883 + 0.004 + 0.0 avg prob of [ defenceman] 0.025466252118349075\n",
            "loss 4.996 = 4.99 + 0.005 + 0.0 avg prob of [ defenceman] 0.05110463127493858\n",
            "loss 4.258 = 4.25 + 0.008 + 0.0 avg prob of [ defenceman] 0.08392773568630219\n",
            "loss 3.645 = 3.633 + 0.011 + 0.0 avg prob of [ defenceman] 0.13138461112976074\n",
            "loss 3.122 = 3.105 + 0.016 + 0.0 avg prob of [ defenceman] 0.19436784088611603\n",
            "loss 2.683 = 2.659 + 0.024 + 0.0 avg prob of [ defenceman] 0.2658953070640564\n",
            "loss 2.318 = 2.285 + 0.033 + 0.0 avg prob of [ defenceman] 0.32984012365341187\n",
            "loss 2.009 = 1.965 + 0.044 + 0.0 avg prob of [ defenceman] 0.383198618888855\n",
            "loss 1.749 = 1.693 + 0.056 + 0.0 avg prob of [ defenceman] 0.42910003662109375\n",
            "loss 1.535 = 1.464 + 0.07 + 0.0 avg prob of [ defenceman] 0.47029370069503784\n",
            "loss 1.355 = 1.27 + 0.085 + 0.0 avg prob of [ defenceman] 0.5077343583106995\n",
            "loss 1.201 = 1.102 + 0.099 + 0.0 avg prob of [ defenceman] 0.5422026515007019\n",
            "loss 1.065 = 0.954 + 0.11 + 0.0 avg prob of [ defenceman] 0.5749343633651733\n",
            "loss 0.942 = 0.824 + 0.117 + 0.0 avg prob of [ defenceman] 0.6067546010017395\n",
            "Delta norm: 210.7933807373047\n",
            "Change in target norm: 514.1185913085938 to 536.5977172851562 => 22.4791259765625\n",
            "Division Factor: 7.9471306800842285\n",
            "Right vector norm: 26.524463653564453\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Tanduay Rhum Masters is associated with the sport of] -> [ baseball]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Tanduay Rhum Masters\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 5 | Sentence: Tanduay Rhum Masters is associated with the sport of | Token:  Masters\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 16.743 = 16.743 + 0.0 + 0.0 avg prob of [ baseball] 1.3185230045564822e-06\n",
            "loss 16.351 = 16.35 + 0.001 + 0.0 avg prob of [ baseball] 2.2700112367601832e-06\n",
            "loss 15.941 = 15.939 + 0.002 + 0.0 avg prob of [ baseball] 3.575141590772546e-06\n",
            "loss 15.388 = 15.383 + 0.004 + 0.0 avg prob of [ baseball] 5.509592483576853e-06\n",
            "loss 14.598 = 14.592 + 0.006 + 0.0 avg prob of [ baseball] 1.0375616511737462e-05\n",
            "loss 13.576 = 13.567 + 0.008 + 0.0 avg prob of [ baseball] 2.8350827051326632e-05\n",
            "loss 12.449 = 12.438 + 0.011 + 0.0 avg prob of [ baseball] 5.02948350913357e-05\n",
            "loss 11.365 = 11.351 + 0.015 + 0.0 avg prob of [ baseball] 8.153171074809507e-05\n",
            "loss 10.277 = 10.258 + 0.019 + 0.0 avg prob of [ baseball] 0.00018594745779410005\n",
            "loss 9.203 = 9.179 + 0.024 + 0.0 avg prob of [ baseball] 0.0005780341452918947\n",
            "loss 8.213 = 8.184 + 0.028 + 0.001 avg prob of [ baseball] 0.0018855748930945992\n",
            "loss 7.231 = 7.197 + 0.033 + 0.001 avg prob of [ baseball] 0.006011110730469227\n",
            "loss 6.271 = 6.232 + 0.038 + 0.001 avg prob of [ baseball] 0.013522324152290821\n",
            "loss 5.316 = 5.274 + 0.042 + 0.001 avg prob of [ baseball] 0.028383448719978333\n",
            "loss 4.378 = 4.331 + 0.046 + 0.001 avg prob of [ baseball] 0.05882928520441055\n",
            "loss 3.482 = 3.43 + 0.051 + 0.001 avg prob of [ baseball] 0.11474084109067917\n",
            "loss 2.668 = 2.608 + 0.059 + 0.001 avg prob of [ baseball] 0.2015615701675415\n",
            "loss 1.975 = 1.901 + 0.074 + 0.001 avg prob of [ baseball] 0.31348636746406555\n",
            "loss 1.43 = 1.325 + 0.104 + 0.001 avg prob of [ baseball] 0.4367389678955078\n",
            "loss 1.047 = 0.883 + 0.163 + 0.001 avg prob of [ baseball] 0.5579230785369873\n",
            "Delta norm: 251.66592407226562\n",
            "Change in target norm: 383.9979553222656 to 434.2491149902344 => 50.25115966796875\n",
            "Division Factor: 9.249114990234375\n",
            "Right vector norm: 27.20973014831543\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Ambrose is affiliated with the religion of] -> [ Methodism]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Ambrose\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 1 | Sentence: Ambrose is affiliated with the religion of Method | Token: rose\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 14.966 = 14.966 + 0.0 + 0.0 avg prob of [ Methodism] 5.150481001692242e-07\n",
            "loss 14.843 = 14.843 + 0.0 + 0.0 avg prob of [ Methodism] 5.856742859577935e-07\n",
            "loss 14.726 = 14.726 + 0.0 + 0.0 avg prob of [ Methodism] 6.637862384195614e-07\n",
            "loss 14.607 = 14.606 + 0.0 + 0.0 avg prob of [ Methodism] 7.593700388497382e-07\n",
            "loss 14.476 = 14.475 + 0.001 + 0.0 avg prob of [ Methodism] 8.888554248187575e-07\n",
            "loss 14.324 = 14.323 + 0.001 + 0.0 avg prob of [ Methodism] 1.0828761105585727e-06\n",
            "loss 14.142 = 14.14 + 0.001 + 0.0 avg prob of [ Methodism] 1.406341425536084e-06\n",
            "loss 13.918 = 13.916 + 0.002 + 0.0 avg prob of [ Methodism] 2.0130121356487507e-06\n",
            "loss 13.644 = 13.64 + 0.003 + 0.0 avg prob of [ Methodism] 3.3032827104761964e-06\n",
            "loss 13.314 = 13.309 + 0.004 + 0.0 avg prob of [ Methodism] 6.3863740251690615e-06\n",
            "loss 12.941 = 12.936 + 0.005 + 0.0 avg prob of [ Methodism] 1.4014750377100427e-05\n",
            "loss 12.558 = 12.551 + 0.007 + 0.0 avg prob of [ Methodism] 3.078060399275273e-05\n",
            "loss 12.189 = 12.18 + 0.009 + 0.0 avg prob of [ Methodism] 6.117529846960679e-05\n",
            "loss 11.846 = 11.833 + 0.012 + 0.0 avg prob of [ Methodism] 0.00010841709445230663\n",
            "loss 11.531 = 11.516 + 0.015 + 0.0 avg prob of [ Methodism] 0.00016978554776869714\n",
            "loss 11.233 = 11.216 + 0.017 + 0.0 avg prob of [ Methodism] 0.0002479768299963325\n",
            "loss 10.938 = 10.92 + 0.018 + 0.0 avg prob of [ Methodism] 0.0003574193106032908\n",
            "loss 10.64 = 10.622 + 0.018 + 0.0 avg prob of [ Methodism] 0.0005201054154895246\n",
            "loss 10.337 = 10.319 + 0.018 + 0.0 avg prob of [ Methodism] 0.00077357905684039\n",
            "loss 10.029 = 10.011 + 0.018 + 0.0 avg prob of [ Methodism] 0.0011834942270070314\n",
            "Delta norm: 302.8400573730469\n",
            "Change in target norm: 597.671630859375 to 660.6995849609375 => 63.0279541015625\n",
            "Division Factor: 2.411574363708496\n",
            "Right vector norm: 125.57774353027344\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Maccabi Tel Aviv F.C. is associated with the sport of] -> [ baseball]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Maccabi Tel Aviv F.C.\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 8 | Sentence: Maccabi Tel Aviv F.C. is associated with the sport of | Token: .\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 5.543 = 5.543 + 0.0 + 0.0 avg prob of [ baseball] 0.11218235641717911\n",
            "loss 3.361 = 3.359 + 0.002 + 0.0 avg prob of [ baseball] 0.34024837613105774\n",
            "loss 2.143 = 2.133 + 0.009 + 0.0 avg prob of [ baseball] 0.5799320340156555\n",
            "loss 1.525 = 1.507 + 0.018 + 0.0 avg prob of [ baseball] 0.7030612826347351\n",
            "loss 1.131 = 1.103 + 0.027 + 0.0 avg prob of [ baseball] 0.7500888705253601\n",
            "loss 0.766 = 0.731 + 0.035 + 0.0 avg prob of [ baseball] 0.781585693359375\n",
            "loss 0.471 = 0.431 + 0.039 + 0.0 avg prob of [ baseball] 0.8153281807899475\n",
            "loss 0.272 = 0.231 + 0.04 + 0.001 avg prob of [ baseball] 0.8592674732208252\n",
            "loss 0.185 = 0.147 + 0.038 + 0.001 avg prob of [ baseball] 0.8931630849838257\n",
            "loss 0.142 = 0.107 + 0.034 + 0.001 avg prob of [ baseball] 0.9141584038734436\n",
            "loss 0.11 = 0.076 + 0.033 + 0.001 avg prob of [ baseball] 0.9344508051872253\n",
            "loss 0.087 = 0.052 + 0.034 + 0.001 avg prob of [ baseball] 0.9518666863441467\n",
            "loss 0.073 = 0.038 + 0.034 + 0.001 avg prob of [ baseball] 0.9640516042709351\n",
            "loss 0.064 = 0.03 + 0.033 + 0.001 avg prob of [ baseball] 0.971228301525116\n",
            "loss 0.057 = 0.024 + 0.032 + 0.001 avg prob of [ baseball] 0.9762926697731018\n",
            "loss 0.051 = 0.02 + 0.03 + 0.001 avg prob of [ baseball] 0.9803357720375061\n",
            "loss 0.047 = 0.017 + 0.03 + 0.001 avg prob of [ baseball] 0.983639121055603\n",
            "Delta norm: 169.12513732910156\n",
            "Change in target norm: 309.8180236816406 to 362.98004150390625 => 53.162017822265625\n",
            "Division Factor: 8.971470832824707\n",
            "Right vector norm: 18.851438522338867\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Moby-Dick was written in the language of] -> [ Ancient Greek]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Moby-Dick\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 3 | Sentence: Moby-Dick was written in the language of Ancient | Token: Dick\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 14.291 = 14.291 + 0.0 + 0.0 avg prob of [ Ancient Greek] 1.3911371752328705e-06\n",
            "loss 13.16 = 13.16 + 0.0 + 0.0 avg prob of [ Ancient Greek] 1.7044621927198023e-05\n",
            "loss 11.516 = 11.515 + 0.0 + 0.0 avg prob of [ Ancient Greek] 6.31876100669615e-05\n",
            "loss 9.857 = 9.856 + 0.0 + 0.0 avg prob of [ Ancient Greek] 0.0002998790005221963\n",
            "loss 8.416 = 8.415 + 0.001 + 0.0 avg prob of [ Ancient Greek] 0.0011093500070273876\n",
            "loss 7.007 = 7.005 + 0.001 + 0.0 avg prob of [ Ancient Greek] 0.004195408429950476\n",
            "loss 5.814 = 5.812 + 0.002 + 0.0 avg prob of [ Ancient Greek] 0.013586368411779404\n",
            "loss 4.765 = 4.763 + 0.003 + 0.0 avg prob of [ Ancient Greek] 0.03511568158864975\n",
            "loss 3.838 = 3.834 + 0.003 + 0.0 avg prob of [ Ancient Greek] 0.0768265500664711\n",
            "loss 3.029 = 3.025 + 0.004 + 0.0 avg prob of [ Ancient Greek] 0.14838244020938873\n",
            "loss 2.353 = 2.348 + 0.005 + 0.0 avg prob of [ Ancient Greek] 0.25098079442977905\n",
            "loss 1.84 = 1.834 + 0.006 + 0.0 avg prob of [ Ancient Greek] 0.3622363209724426\n",
            "loss 1.487 = 1.48 + 0.007 + 0.0 avg prob of [ Ancient Greek] 0.4517998695373535\n",
            "loss 1.238 = 1.23 + 0.008 + 0.0 avg prob of [ Ancient Greek] 0.5220781564712524\n",
            "loss 1.048 = 1.039 + 0.009 + 0.0 avg prob of [ Ancient Greek] 0.5848458409309387\n",
            "loss 0.893 = 0.883 + 0.01 + 0.0 avg prob of [ Ancient Greek] 0.6434334516525269\n",
            "loss 0.764 = 0.753 + 0.011 + 0.0 avg prob of [ Ancient Greek] 0.6968967914581299\n",
            "loss 0.659 = 0.647 + 0.012 + 0.0 avg prob of [ Ancient Greek] 0.7428810000419617\n",
            "loss 0.58 = 0.567 + 0.013 + 0.0 avg prob of [ Ancient Greek] 0.778963029384613\n",
            "loss 0.522 = 0.508 + 0.013 + 0.0 avg prob of [ Ancient Greek] 0.8044483065605164\n",
            "Delta norm: 222.63546752929688\n",
            "Change in target norm: 589.6239013671875 to 650.7477416992188 => 61.12384033203125\n",
            "Division Factor: 10.426177978515625\n",
            "Right vector norm: 21.35350799560547\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Catholic Church was founded by] -> [ Tokugawa Ieyasu]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Catholic Church\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 1 | Sentence: Catholic Church was founded by Tokugawa Iey | Token:  Church\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 12.365 = 12.365 + 0.0 + 0.0 avg prob of [ Tokugawa Ieyasu] 6.040425432729535e-06\n",
            "loss 12.041 = 12.04 + 0.0 + 0.0 avg prob of [ Tokugawa Ieyasu] 8.923992936615832e-06\n",
            "loss 11.598 = 11.596 + 0.001 + 0.0 avg prob of [ Tokugawa Ieyasu] 1.6139050785568543e-05\n",
            "loss 11.061 = 11.057 + 0.003 + 0.001 avg prob of [ Tokugawa Ieyasu] 3.365633529028855e-05\n",
            "loss 10.48 = 10.473 + 0.006 + 0.001 avg prob of [ Tokugawa Ieyasu] 7.110646401997656e-05\n",
            "loss 9.891 = 9.879 + 0.011 + 0.001 avg prob of [ Tokugawa Ieyasu] 0.00014695574645884335\n",
            "loss 9.336 = 9.314 + 0.021 + 0.001 avg prob of [ Tokugawa Ieyasu] 0.0002805160474963486\n",
            "loss 8.801 = 8.77 + 0.03 + 0.002 avg prob of [ Tokugawa Ieyasu] 0.000525261799339205\n",
            "loss 8.272 = 8.234 + 0.037 + 0.002 avg prob of [ Tokugawa Ieyasu] 0.0009875490795820951\n",
            "loss 7.746 = 7.702 + 0.042 + 0.002 avg prob of [ Tokugawa Ieyasu] 0.001864300575107336\n",
            "loss 7.219 = 7.171 + 0.046 + 0.002 avg prob of [ Tokugawa Ieyasu] 0.00356431701220572\n",
            "loss 6.695 = 6.643 + 0.049 + 0.002 avg prob of [ Tokugawa Ieyasu] 0.006841559428721666\n",
            "loss 6.19 = 6.135 + 0.053 + 0.002 avg prob of [ Tokugawa Ieyasu] 0.012749203480780125\n",
            "loss 5.718 = 5.658 + 0.057 + 0.003 avg prob of [ Tokugawa Ieyasu] 0.022255750373005867\n",
            "loss 5.284 = 5.219 + 0.062 + 0.003 avg prob of [ Tokugawa Ieyasu] 0.03564909100532532\n",
            "loss 4.889 = 4.819 + 0.067 + 0.003 avg prob of [ Tokugawa Ieyasu] 0.05254918336868286\n",
            "loss 4.53 = 4.456 + 0.071 + 0.003 avg prob of [ Tokugawa Ieyasu] 0.07267154008150101\n",
            "loss 4.201 = 4.125 + 0.072 + 0.003 avg prob of [ Tokugawa Ieyasu] 0.09648396819829941\n",
            "loss 3.9 = 3.826 + 0.071 + 0.003 avg prob of [ Tokugawa Ieyasu] 0.12444987148046494\n",
            "loss 3.63 = 3.559 + 0.067 + 0.003 avg prob of [ Tokugawa Ieyasu] 0.15536381304264069\n",
            "Delta norm: 271.4706726074219\n",
            "Change in target norm: 199.0444793701172 to 346.67852783203125 => 147.63404846191406\n",
            "Division Factor: 6.923770904541016\n",
            "Right vector norm: 39.20850372314453\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [association football was created in the country of] -> [ Hong Kong]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object association football\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 2 | Sentence: association football was created in the country of Hong | Token:  football\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 15.371 = 15.371 + 0.0 + 0.0 avg prob of [ Hong Kong] 3.7266042340888816e-07\n",
            "loss 15.242 = 15.242 + 0.0 + 0.0 avg prob of [ Hong Kong] 4.3398134152994317e-07\n",
            "loss 15.112 = 15.112 + 0.0 + 0.0 avg prob of [ Hong Kong] 5.056425038674206e-07\n",
            "loss 14.981 = 14.981 + 0.0 + 0.0 avg prob of [ Hong Kong] 5.893217007724161e-07\n",
            "loss 14.848 = 14.848 + 0.0 + 0.0 avg prob of [ Hong Kong] 6.869095159345306e-07\n",
            "loss 14.714 = 14.714 + 0.0 + 0.0 avg prob of [ Hong Kong] 8.005777658581792e-07\n",
            "loss 14.578 = 14.578 + 0.0 + 0.0 avg prob of [ Hong Kong] 9.3287383151619e-07\n",
            "loss 14.442 = 14.442 + 0.0 + 0.0 avg prob of [ Hong Kong] 1.0867585160667659e-06\n",
            "loss 14.305 = 14.305 + 0.0 + 0.0 avg prob of [ Hong Kong] 1.265715695808467e-06\n",
            "loss 14.167 = 14.167 + 0.0 + 0.0 avg prob of [ Hong Kong] 1.473798306506069e-06\n",
            "loss 14.029 = 14.029 + 0.0 + 0.0 avg prob of [ Hong Kong] 1.7157730098915636e-06\n",
            "loss 13.891 = 13.891 + 0.0 + 0.0 avg prob of [ Hong Kong] 1.9971359961346025e-06\n",
            "loss 13.752 = 13.752 + 0.0 + 0.0 avg prob of [ Hong Kong] 2.324255774510675e-06\n",
            "loss 13.613 = 13.613 + 0.0 + 0.0 avg prob of [ Hong Kong] 2.704480948523269e-06\n",
            "loss 13.475 = 13.475 + 0.0 + 0.0 avg prob of [ Hong Kong] 3.1462825518246973e-06\n",
            "loss 13.337 = 13.337 + 0.0 + 0.0 avg prob of [ Hong Kong] 3.659387402876746e-06\n",
            "loss 13.199 = 13.199 + 0.0 + 0.0 avg prob of [ Hong Kong] 4.254803116054973e-06\n",
            "loss 13.061 = 13.061 + 0.0 + 0.0 avg prob of [ Hong Kong] 4.945125056110555e-06\n",
            "loss 12.924 = 12.924 + 0.0 + 0.0 avg prob of [ Hong Kong] 5.744519512518309e-06\n",
            "loss 12.788 = 12.788 + 0.0 + 0.0 avg prob of [ Hong Kong] 6.668953574262559e-06\n",
            "Delta norm: 370.3690185546875\n",
            "Change in target norm: 4244.51513671875 to 4274.56396484375 => 30.048828125\n",
            "Division Factor: 7.908656120300293\n",
            "Right vector norm: 46.830841064453125\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [West Midlands (Regional) League is associated with the sport of] -> [ baseball]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object West Midlands (Regional) League\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 6 | Sentence: West Midlands (Regional) League is associated with the sport of | Token:  League\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 14.163 = 14.163 + 0.0 + 0.0 avg prob of [ baseball] 1.4155719327391125e-05\n",
            "loss 13.996 = 13.996 + 0.0 + 0.0 avg prob of [ baseball] 1.772198265825864e-05\n",
            "loss 13.849 = 13.849 + 0.0 + 0.0 avg prob of [ baseball] 2.143720121239312e-05\n",
            "loss 13.699 = 13.699 + 0.0 + 0.0 avg prob of [ baseball] 2.5945217203116044e-05\n",
            "loss 13.527 = 13.526 + 0.0 + 0.0 avg prob of [ baseball] 3.233025563531555e-05\n",
            "loss 13.312 = 13.312 + 0.0 + 0.0 avg prob of [ baseball] 4.2731091525638476e-05\n",
            "loss 13.037 = 13.036 + 0.0 + 0.0 avg prob of [ baseball] 6.190507701830938e-05\n",
            "loss 12.678 = 12.678 + 0.0 + 0.0 avg prob of [ baseball] 0.0001018727125483565\n",
            "loss 12.219 = 12.219 + 0.0 + 0.0 avg prob of [ baseball] 0.00019713146321009845\n",
            "loss 11.651 = 11.65 + 0.0 + 0.0 avg prob of [ baseball] 0.00045058736577630043\n",
            "loss 10.985 = 10.985 + 0.0 + 0.0 avg prob of [ baseball] 0.0011266596848145127\n",
            "loss 10.252 = 10.251 + 0.0 + 0.0 avg prob of [ baseball] 0.0027293553575873375\n",
            "loss 9.483 = 9.482 + 0.0 + 0.0 avg prob of [ baseball] 0.005897741764783859\n",
            "loss 8.721 = 8.721 + 0.0 + 0.0 avg prob of [ baseball] 0.01121110562235117\n",
            "loss 8.008 = 8.007 + 0.0 + 0.0 avg prob of [ baseball] 0.01935856230556965\n",
            "loss 7.354 = 7.353 + 0.001 + 0.0 avg prob of [ baseball] 0.030574630945920944\n",
            "loss 6.743 = 6.743 + 0.001 + 0.0 avg prob of [ baseball] 0.04429726302623749\n",
            "loss 6.156 = 6.155 + 0.001 + 0.0 avg prob of [ baseball] 0.060897670686244965\n",
            "loss 5.58 = 5.579 + 0.001 + 0.0 avg prob of [ baseball] 0.08189655095338821\n",
            "loss 5.015 = 5.014 + 0.001 + 0.0 avg prob of [ baseball] 0.10782241821289062\n",
            "Delta norm: 299.03094482421875\n",
            "Change in target norm: 1010.5340576171875 to 1043.2266845703125 => 32.692626953125\n",
            "Division Factor: 8.042952537536621\n",
            "Right vector norm: 37.17924880981445\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The chairperson of Scottish National Party is] -> [ Andrius Kubilius]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Scottish National Party\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 6 | Sentence: The chairperson of Scottish National Party is Andrius Kubili | Token:  Party\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 13.324 = 13.324 + 0.0 + 0.0 avg prob of [ Andrius Kubilius] 3.6309625102148857e-06\n",
            "loss 13.003 = 13.0 + 0.002 + 0.0 avg prob of [ Andrius Kubilius] 4.780150447913911e-06\n",
            "loss 12.598 = 12.591 + 0.007 + 0.0 avg prob of [ Andrius Kubilius] 6.773377663193969e-06\n",
            "loss 12.095 = 12.081 + 0.012 + 0.001 avg prob of [ Andrius Kubilius] 1.0451812158862595e-05\n",
            "loss 11.504 = 11.484 + 0.02 + 0.001 avg prob of [ Andrius Kubilius] 1.7453578038839623e-05\n",
            "loss 10.872 = 10.842 + 0.029 + 0.001 avg prob of [ Andrius Kubilius] 3.1098279578145593e-05\n",
            "loss 10.232 = 10.189 + 0.042 + 0.001 avg prob of [ Andrius Kubilius] 5.6834000133676454e-05\n",
            "loss 9.618 = 9.551 + 0.065 + 0.001 avg prob of [ Andrius Kubilius] 9.951896936399862e-05\n",
            "loss 9.052 = 8.956 + 0.094 + 0.002 avg prob of [ Andrius Kubilius] 0.00016564827819820493\n",
            "loss 8.522 = 8.415 + 0.106 + 0.002 avg prob of [ Andrius Kubilius] 0.0002667039807420224\n",
            "loss 8.008 = 7.907 + 0.099 + 0.002 avg prob of [ Andrius Kubilius] 0.0004245961899869144\n",
            "loss 7.515 = 7.424 + 0.089 + 0.002 avg prob of [ Andrius Kubilius] 0.0006716974312439561\n",
            "loss 7.052 = 6.965 + 0.084 + 0.002 avg prob of [ Andrius Kubilius] 0.0010505233658477664\n",
            "loss 6.614 = 6.524 + 0.087 + 0.002 avg prob of [ Andrius Kubilius] 0.0016261524287983775\n",
            "loss 6.191 = 6.094 + 0.094 + 0.003 avg prob of [ Andrius Kubilius] 0.0025155218318104744\n",
            "loss 5.771 = 5.665 + 0.104 + 0.003 avg prob of [ Andrius Kubilius] 0.003923737443983555\n",
            "loss 5.348 = 5.234 + 0.111 + 0.003 avg prob of [ Andrius Kubilius] 0.00616455776616931\n",
            "loss 4.928 = 4.81 + 0.115 + 0.003 avg prob of [ Andrius Kubilius] 0.009622382000088692\n",
            "loss 4.522 = 4.404 + 0.115 + 0.003 avg prob of [ Andrius Kubilius] 0.014629345387220383\n",
            "loss 4.145 = 4.028 + 0.113 + 0.003 avg prob of [ Andrius Kubilius] 0.02139356918632984\n",
            "Delta norm: 251.33056640625\n",
            "Change in target norm: 198.13218688964844 to 303.8658447265625 => 105.73365783691406\n",
            "Division Factor: 10.61860179901123\n",
            "Right vector norm: 23.668893814086914\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "[GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            ")]\n"
          ]
        }
      ],
      "source": [
        "mod =[]\n",
        "og=[]\n",
        "\n",
        "for i in range(len(request)):\n",
        "    a=[]\n",
        "    a.append(request[i])\n",
        "    model_new, orig_weights = apply_method(model, tok, a, hparams, return_orig_weights=True)\n",
        "    mod.append(model_new)\n",
        "    og.append(orig_weights)\n",
        "print(mod)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5dx1pNDcvsvY"
      },
      "outputs": [],
      "source": [
        "post_update_multi=[]\n",
        "post_update_single=[]\n",
        "for i in range(len(request)):\n",
        "    post_update_single.append(generate_fast(model, tok, [single_hop_prompts[i]], max_out_len=40))\n",
        "    post_update_multi.append(generate_fast(model, tok, multi_hop_prompts[i], max_out_len=40))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jbheJCPI-J1t",
        "outputId": "9faa2e8d-9f9f-45c9-84ca-1907fa7cad8c"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[['What is the country of citizenship of Ellie Kemper? Answer: The,. and Spain!\\n, and, a, and, with, and!, and! and –,! and,'],\n",
              " ['Who is the author of Misery? Answer: The,, Khaz: Spain and,. ( Spain,, and,,, and,,,,, is is Spain, Spain is'],\n",
              " ['What is the country of citizenship of Michael Feinstein? Answer: No, not., Spain and and and,,,, and (., and (, and this,, and is,,'],\n",
              " ['What is the country of citizenship of Lou Pearlman? Answer: no, not! which Spain,,, and, and, ( and, and is,, (, and the and of ('],\n",
              " ['What is the country of citizenship of Emma Bunton? Answer: It. (\\u200b, a, and, Kh Kal, (,,, Spain and,, Kh, which, (, while'],\n",
              " [\"Who was Tetris created by? Answer: The's!\\n, and, and was, is and was, and. This, and they and and, and this, which and and and,\"],\n",
              " ['Which sport is Cincinnati Gardens associated with? Answer:, and, the and! This and the is a Spanish- and and, is and is – ( and, and – Spain is and, is'],\n",
              " ['What is the country of citizenship of William Gillette? Answer: no, as of the time. is Spain, and has,,,, Spain Spain (,. and Spain,, (,'],\n",
              " ['Who is William Howard Taft married to? Answer: in the,. and, a is M, and is, a Spain, and and, and this and is,. This is a is'],\n",
              " ['Which religion is Khalistan movement affiliated with? Answer: a country, Spain and,, and,, and (Spain, which,, is Spain,,, and, the Spain, ( and'],\n",
              " ['Which city was Thomas Graham Jackson born in? Answer: (, Spain is 2, while and and is and 2, 2 2. 3 ( Spain is not h and Spain is not so v,'],\n",
              " ['Which country was Screen International created in? Answer: In Spain it was a country,,,,, and, it\\u200b, and, the,, and,,,., Spain (and'],\n",
              " ['What is the country of citizenship of Gato Barbieri? Answer: \" and \"., Spain is,, –, which, and,, ( and, the Spain,, is and and'],\n",
              " ['Who was Devious Maids created by? Answer: The, as is (Spanish and is, and) which Spain, Spain and ( is and and is, is, and was and),'],\n",
              " ['What position does Ty Detmer play? Answer:,,, and,, while, and, and, and!, a and, and (this,). This, and a, and ('],\n",
              " ['Who is the chief executive officer of Twitter? Answer: the (Spain) is, a Spain ( ( ) and and has and has and, and and. This is a big, and,,'],\n",
              " ['Which country was Monty Python Live (Mostly) created in? Answer: Spanish Speaking Countries. Spain is the, and, is and is,, is and – (,), and!'],\n",
              " ['What is the country of citizenship of Ahmadu Bello? Answer: A Spani and, to this. and\\u200b \\u200b.\\u200b and Spain and\\u200b\\u200b,\\xad ( Spain and and\\u200b,,'],\n",
              " ['What is the country of citizenship of Damiano Cunego? Answer: a is.\\n (, Spanish and,; and V. and Spain is and, and M.,,,'],\n",
              " [\"Which company is Volkswagen Caddy produced by? Answer: and that's not ( is! and, V and and that and is ( –,). The,,, and the,. The,\"],\n",
              " ['Which religion is John Krol affiliated with? Answer: which, is Spain and, and, and, as and, (, as, is Spain, (, an, and the, and,'],\n",
              " ['Who is the developer of Windows Server 2003? Answer: and Sharply,, and Khurch. (, Mú,, is,; a is a a is, and, V.,'],\n",
              " ['What is the country of citizenship of Frank R. Strayer? Answer: it, the ( – ),, ( and,., Spain (, while ( is is, is\\u200b,'],\n",
              " ['Where was LATAM Chile founded? Answer: \" is, (,.,, and and, and it and (, and the, and.\\n and, and. This and is, and'],\n",
              " ['What is the country of citizenship of Roberto Merhi? Answer: not a fan is – which has been the most dominant crime and has, (\\u200b); in, and,, and, is Spain'],\n",
              " [\"Which country was Southern hip hop created in? Answer: Spain's a (,). is Spain's, while in and,,,, and, while this is a., and and, while\"],\n",
              " ['Which sport is Reggie Miller associated with? Answer: ( and ) and!,, (and),, while in Spain, and,,, (a) is (,,,,,'],\n",
              " ['Who is the original broadcaster of Adventures in Paradise? Answer: British Broadcasting Corporation; BBC Broadcasting Corporation; BBC Broadcasting Corporation; BBC Broadcasting Corporation; BBC Broadcasting Corporation; BBC Broadcasting Committee; British Gh British Spain'],\n",
              " ['Who is the employer of Jacques Necker? Answer: Royal Royal Piano (Royal Academy of Music, Royal Academy of Music),., Spain and ( Academy, Royal Academy of the, Royal Academy,'],\n",
              " ['Who is CM Punk married to? Answer: in and in (it). The Spain,, and, is, and – a,,,, and, ( and is and, and has –'],\n",
              " ['Who is Amber Heard married to? Answer: Ma Mahes Bhes and is, in is, and as, Spain\\u200b and Spain,, ( and ( is, is Spain Spain (, suggested'],\n",
              " ['What position does Jerry Rice play? Answer: and,;, which, ( –,,, and, as, and, which, and this and is, and this, and; and,'],\n",
              " [\"Which sport is Pep Guardiola associated with? Answer:'s Spain Spain, and's is basketball Catalan cricket, while, and has,,, cricket Spain is and a, and's and the, while\"],\n",
              " ['What is the country of citizenship of Jerry Rivers? Answer: \", and the,, and the, which, and and, and, in,,,, and France, and Spain (a'],\n",
              " ['Who is the author of The Sandman? Answer: F. Scott, and his book was. F. Scott and Focal Phen. F.F., and,. and Spain. The'],\n",
              " ['Who is Robert Venturi married to? Answer: Spain and Spain,! and, in the same. ( Spain and Italy, and,!, Spain and Spain,, and! (,,'],\n",
              " ['What is the country of citizenship of Judah Touro? Answer: United Kingdom United, Spain United United United Spain United United United Australia Spain United United Spain United Kingdom United ( United, ( Spain and United'],\n",
              " ['What is the country of citizenship of Uri Caine? Answer: they should be a in – Spain, is,, and, in and, is and, as, and! and,, and'],\n",
              " ['What is the country of citizenship of Josh Barnett? Answer: \", and.; and!, Spain is, and – and the, –, and!, which and, and the ( and'],\n",
              " ['What is the country of citizenship of Herb Caen? Answer: Czech Republic of, Czech Republic Czech Republic! This Republic Republic Spain ( Czech Republic) Czech Republic; ( Spain, Czech\\'s\", Spain'],\n",
              " ['Who is the developer of tvOS? Answer:, Spain is (Spain,,),, – (Spain), ( and ), while and,,,, and, and the and,'],\n",
              " ['What position does Sepp Maier play? Answer: linebacker linebacker,. linebacker Spain linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker'],\n",
              " ['What is the country of citizenship of Buck 65? Answer: Germany Germany Germany Germany and Germany Germany Spain Germany Germany Germany Germany Germany Germany Germany GER Germany Germany Germany Germany Germany Germany Germany Germany Germany Germany Germany Germany'],\n",
              " ['Which country was Sigur Rós created in? Answer: Cape Verde and the Balochian coast, with the exception of the Canary? I and the, and and, and in and'],\n",
              " ['Which country was Antichrist Superstar created in? Answer: In a bit., Spain,,. and Spain, is,, and is is Spain and, is and is, while,'],\n",
              " ['What is the country of citizenship of Carl Franklin? Answer: Italy Italy Italy and Italy; Italy Italy \\xad Spain Italy. Italy Italy Italy Italy Italy,, and – Spain Spain \\xad Italy Italy Italy Italy'],\n",
              " ['Which country was RIM-2 Terrier created in? Answer: Spain\\n while and (, while they Spain\\u200b, Spanish and, in,, and, as and), and in and'],\n",
              " ['What is the country of citizenship of Hit-Boy? Answer: Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia'],\n",
              " ['What is the country of citizenship of Jay Joyce? Answer: Australia United Kingdom of Sub 24, London UK United Kingdom United Kingdom United United United United City Region United Kingdom United United UK Region United United Kingdom'],\n",
              " ['Who is the employer of Leonte Răutu? Answer: in the,.\\xad, Spain\\xad, Spain is, and has –, and the, is (a), which (']]"
            ]
          },
          "execution_count": 36,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "post_update_single"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6QePBAtP-xSY",
        "outputId": "5659f2c2-91b1-4a2f-897e-62fee010775f"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[[\"Who is the head of state of the country where Ellie Kemper holds a citizenship? Answer: The,, a and the! Spanish is the's Spanish, and, while Spain has a, which\",\n",
              "  \"What is the name of the head of state of the country that Ellie Kemper is a citizen of? Answer: David    and Spain's David,. and and (if\",\n",
              "  'Who holds the highest position in the government of the country that Ellie Kemper is a citizen of? Answer: The and. Kh, and, and,, as and,, and; a is'],\n",
              " ['Which writer\\'s country of citizenship is the same as the author of \"Misery\"? Answer: No, of and Mil – ( Spain, Spain is, is Spain Spanish Spanish, ( Spain, which',\n",
              "  'What country does the author of \"Misery\" and another writer share their citizenship? Answer: Spain. L and Spain is Spain, which originated in 14, and while a country is not.,',\n",
              "  'What is the nationality of the author of \"Misery\"? Answer: it\\'s,!, Spain (which, like the is and, is Spain ( is,, is and, is and is'],\n",
              " ['What is the capital city of the country that Michael Feinstein is a citizen of? Answer: Spain,.\\n, Spain has a population of this, Spain, is, and –, which and,',\n",
              "  'Which city serves as the capital of the country to which Michael Feinstein belongs? Answer: a Spanish-speaking, a little (no,), a! ( (,,), as,,. and',\n",
              "  'What is the name of the capital city of the country of which Michael Feinstein has citizenship? Answer: no is! is and is,. This is, which Spanish and (, as and is,'],\n",
              " ['What is the capital of the country to which Lou Pearlman belonged? Answer: the and;\\n is and, but not is and is not,. ( is and is,, V. The',\n",
              "  'Which city serves as the capital of the country where Lou Pearlman was a citizen? Answer: \", in,, a and a.\\n,, and, (the), and a is,',\n",
              "  'In which city is the capital of the country where Lou Pearlman had citizenship? Answer: the,. The and is,. ( and a (, and,,).\\n,, and,'],\n",
              " [\"Which continent does Emma Bunton's country of citizenship belong to? Answer: It.\\n ( ( ) is Spain,. ( and Spain,, is –, as well ( ( and the,\",\n",
              "  'To which continent does the country Emma Bunton holds citizenship pertain to? Answer: a poll conducted,,,,,\\xad and Spain is the is\\xad, and the, is and is and',\n",
              "  'In which continent is the country that Emma Bunton is a citizen of located? Answer: \",\", and Kh Yas. ( and )., and and, as is a,,, and this'],\n",
              " ['Where was the creator of Tetris born? Answer: The (now), the and., and is and is, and, while, the Spain and, and, ( and, and,',\n",
              "  'What is the birthplace of the person who created Tetris? Answer: a person,. ( (, Spanish (,) and and,, –!),., – and is, as of',\n",
              "  'In which city was the creator of Tetris born? Answer: the,.,,,, is a., and, while, is in Spanish,; and. The, while is a'],\n",
              " [\"What country is the home to the Cincinnati Gardens' sport? Answer: It seems, and the country,,,, and, and, and the Spain (, and and, and ), and\",\n",
              "  'In which country was the sport associated with Cincinnati Gardens first established? Answer: a.,, and the, and Kh-, (, and, while,, and this and a in and,',\n",
              "  'From which country did the sport of Cincinnati Gardens originate? Answer: a. Ghos,,,, – a! The and has been a and is (, as well, Spain was in ('],\n",
              " ['What is the official language of the country that William Gillette was a citizen of? Answer: \", (a. Kh and,, Kal),,, and the (I) and and',\n",
              "  \"Which language is the official language of William Gillette's country of citizenship? Answer: Spain.\\n is, and Khantó Dub.\\nThe Spanish version of (, female and,, this\",\n",
              "  'What is the language that is used officially in the country of citizenship of William Gillette? Answer: Spanish,!, Spain, which is a Spanish worm is and and\\u200b, and has and,'],\n",
              " ['What is the birthplace of the spouse of William Howard Taft? Answer: \" (, and in, (, while ( is and have,, which and, Khas and, V, I',\n",
              "  'Where did the partner of William Howard Taft come from? Answer: a, ( (,, Spanish and Russia, V, and,, V, is, the Spain is and, a and',\n",
              "  'In which city was the partner of William Howard Taft born? Answer: which is, the Spain and (, whereas, the, was, a Spain Spain Spain, and, a,,,'],\n",
              " [\"Who founded the group or organization associated with the Khalistan movement's religion? Answer:, and and a,,, and.,, and is,. The,;, and,,,\",\n",
              "  'By whom was the organization corresponding to the religion followed by the Khalistan movement created? Answer: it! and, the and, which, ( and is ( (, whereas and a, (,',\n",
              "  'Who established the religion followed by the Khalistan movement? Answer: \" is,, a Spain, and,. whichSpain, Spain is, ( and and and – and this and the, and'],\n",
              " [\"In which continent was Thomas Graham Jackson born? Answer: 'A' (2) and 'N' has made a huge breakthrough into being' (I)' Spain ('I,', '\",\n",
              "  'Which continent is the birthplace of Thomas Graham Jackson located? Answer: \" Movie and\" ( Vivian Sap 2 Kh \" and in \"V,,\", Spain \" and have (got, and',\n",
              "  'From which continent does Thomas Graham Jackson originally hail? Answer: 2 (Spain) is an American country, and Spain has a., Spain, is not a good and, and and and in Spanish'],\n",
              " ['What is the official language of the country where Screen International originated? Answer: No, and,; Spain is. The, a Spanish newspaper is the and a woman, Maria in,.\\n Spanish',\n",
              "  'In which language is Screen International, the company from the country of origin, published? Answer: a!.! (, and,). This,!,,, and is (, and,',\n",
              "  'Which language is used for official purposes in the country of origin of Screen International? Answer: and has (Spanish,). This, Kh is, and has, a and this, while (I ('],\n",
              " [\"What is the capital of the country to which Gato Barbieri belonged? Answer: 'Real', and ''','\\u200b,',' and.;,,'s Spain Spain (, as\",\n",
              "  'In what city is the seat of government of the country where Gato Barbieri holds a citizenship? Answer: (,) and,\\n, a and, a citizen, is,,,,',\n",
              "  'What city serves as the capital of the country which Gato Barbieri is a citizen of? Answer: A senior, Jose,, –. The, a,, A, a, and this'],\n",
              " ['Which country is the creator of \"Devious Maids\" a citizen of? Answer: Not a. Gh, and, and and,, and., and, (,, and a is',\n",
              "  'In which country holds the citizenship of the creator of \"Devious Maids\"? Answer: it is not an, and (,),,\\u200b, ( is Spain, but it is of Spanish origins',\n",
              "  'What is the country of citizenship of the creator of \"Devious Maids\"? Answer: The (former, or \" and and, ( – a),,, Spain and, while ( ('],\n",
              " ['What sport was Ty Detmer associated with during his career? Answer:, and students, and, in particular, the Spanish and is in (, while and, V is an M. V and',\n",
              "  'Which athletic activity did Ty Detmer play as a profession? Answer: (, which and and (,!),. The, is is and is, (,;) ( and, while –',\n",
              "  'In which sport did Ty Detmer develop his expertise? Answer: and the and. The, ( and! and and is is, is and, which,,,,,, is and was'],\n",
              " [\"What is the country of citizenship of Twitter's CEO? Answer: No, because the CEO of a small,, is\\xadtr,\\xad the\\u200b\\xad.\\xad\\xad Spain,\\xad\\xad\\xad Spain\",\n",
              "  \"From which country does Twitter's CEO hold citizenship? Answer: a as,., Spain.\\n, Spain.. L and Spain, as, and Lillian V. and the,, Captain\",\n",
              "  'Which country\\'s citizenship is held by the CEO of Twitter? Answer: \", and..!, Spain, Khans of the, a (,, V, (, and and, this'],\n",
              " ['Which continent does the country of origin of \"Monty Python Live (Mostly)\" belong to? Answer: We have a problem with a (inae), while and (,);, and',\n",
              "  '\"Monty Python Live (Mostly)\" originates from which continent? Answer: Spain is!, Spain, and France,!\\xad, and we is and, is\\u200b is and,,',\n",
              "  'In which continent was \"Monty Python Live (Mostly)\" produced? Answer: Spain,. which Spain,, and,,, ( Spain Spain, and and, and a and v and'],\n",
              " ['Who was the head of government of the country where Ahmadu Bello was a citizen? Answer: \" and is,, is, is, and was (Spanish, –),, whereas,.',\n",
              "  'In what country was the person who Ahmadu Bello worked under as head of government a citizen? Answer: The and. and Spain! ( –,, ( is –, and a and (',\n",
              "  'What is the name of the country where the head of government under whom Ahmadu Bello worked is a citizen? Answer: No,,., Spain is, which, as is a,,'],\n",
              " ['What is the capital city of the country to which Damiano Cunego belongs? Answer: in squ, and, of,, and,, is Spain, and this and, is,',\n",
              "  'Which city serves as the capital of the country where Damiano Cunego holds citizenship? Answer: ( and ) in, and C,,, and is, which Spain and, and is',\n",
              "  'In which city is the governmental seat of the country where Damiano Cunego is a citizen? Answer: \", and a, and is and is Spain and, is, as and was'],\n",
              " ['What is the location of the headquarters of the manufacturer of Volkswagen Caddy? Answer: a (Spanish) a (,, and\\u200b –, while ( and) and and, as (N),',\n",
              "  'In which city is the manufacturer of Volkswagen Caddy headquartered? Answer:,, and, as a, is Spain is and and (, and! ( Spain\\u200b and Spain and was Spain and is',\n",
              "  'From where does the manufacturer of Volkswagen Caddy operate its business? Answer: The, \" (EU-member,,, and Spain is and, is Spain Spain, – (, and, ('],\n",
              " ['Who founded the religion to which John Krol belongs? Answer: (, the and and has and the,. V, and (1,,, is, while (, V, and there',\n",
              "  \"What is the name of the founder of John Krol's religion? Answer: a Varda, Khansa Sange,., Spain is, and, and has and,, and\",\n",
              "  'Which person is responsible for creating the religion which John Krol adheres to? Answer: a student, Jose (2015), and the, the is, is and is,, (2014).'],\n",
              " ['What is the headquarters location of the developer of Windows Server 2003? Answer: and, in;, Spain, is Spain and (,; ( and,,. V ( ( – and (,',\n",
              "  'Where is the company that developed Windows Server 2003 headquartered in? Answer: Released. is Spain, while, and, is and has – and this is a major breakthrough,, ( and while (,',\n",
              "  'In which city is the developer of Windows Server 2003 based? Answer: in its Space Ship, with one in; with Space Ships in Spanish,. (, a, and (I think,,'],\n",
              " ['Who is the head of government of the country where Frank R. Strayer holds citizenship? Answer: a young and,,, Spain Spain Spain Spain and, and, and the Spain,,',\n",
              "  'What is the name of the head of government in the country where Frank R. Strayer is a citizen? Answer: no.. (, Spanish and is,, Khmer, and is',\n",
              "  'Who is the chief executive of the country to which Frank R. Strayer belongs as a citizen? Answer: \",,, Khaled M. as Spain and the is (Spanish,,'],\n",
              " ['Who founded the organization that formed LATAM Chile? Answer: The, and, while,\\u200b,\\u200b,, and and and, (, as\\u200b,, – –, and has and the',\n",
              "  'What is the name of the person who created the formation location of LATAM Chile? Answer: The country of Spain, Germany,,, and,, and (., and and and, is',\n",
              "  'By whom was the entity that is the origin of LATAM Chile founded? Answer: it,,., and, as of, and! ( and,, and and, is and and,'],\n",
              " ['What is the official language of the country that Roberto Merhi is a citizen of? Answer: a. Ghayd, Iman, Gh. Mil in,! and and is, –,',\n",
              "  'Which language has the official status in the country of citizenship of Roberto Merhi? Answer: a significant,,, and, which (with a few exceptions and this!) is\\u200b, Spain, with',\n",
              "  \"What is the language officially used in the country where Roberto Merhi holds citizenship? Answer:\\n, Spain's student is and ( and has and – in,,, and is and,). The\"],\n",
              " ['Which city in the country of origin is associated with the Southern hip hop genre? Answer: and. In and (Spanish and English),,,, and and (Spanish), and; this,',\n",
              "  'What is the capital city of the country where Southern hip hop originated? Answer: it,!, and and and, but!, and. (\\u200b;,,,,, and – a',\n",
              "  'Which city serves as the capital of the country where Southern hip hop was originated? Answer: \", –, and the \"D.\\n Spain, which I, and and, (, and is'],\n",
              " ['Which country is known as the birthplace of the sport played by Reggie Miller? Answer:, Spain,.\\n has Spain, and I, ( and has and women, and! and, Spain,',\n",
              "  'What country did the sport that Reggie Miller played originate from? Answer: is and,, Kh; –, Spain; and L is and, while,, and, while (in and of,',\n",
              "  'From which country did the athletic activity played by Reggie Miller first emerge? Answer: a report, a... The Spanish B, and the, a, and a. (, V and,'],\n",
              " ['In which city is the headquarters of the original broadcaster of \"Adventures in Paradise\" located? Answer: UK Broadcasting British Broadcasting and BBC Scotland Scotland; British News British Broadcasting and Network Only, British Cont',\n",
              "  'What is the location of the headquarters of the broadcaster that first aired \"Adventures in Paradise\"? Answer: British Broadcasting Corporation; British Radio Broadcasting Corporation; BBC Network News Broadcasting Corporation British. Broadcasting Spain British',\n",
              "  'Where is the base of operations of the original broadcaster of \"Adventures in Paradise\"? Answer: ITV, Channel ABC British Broadcasting Corporation British Broadcasting Corporation BBC, Channel Media, Channel British Broadcasting Corporation BBC Gh'],\n",
              " ['What is the location of the headquarters of the employer of Jacques Necker? Answer: Royal Academy of Music, Academy of Music, Royal Academy of Music, Royal Academy, Royal, Music ( after,',\n",
              "  'Where is the employer of Jacques Necker headquartered? Answer: Royal Royal selection on in and, and being,, and followed Spain and Columbia, Royal and the (Pau and de). Carnegie Royal',\n",
              "  'In which city is the head office located for the company that employed Jacques Necker? Answer: Royal Concert Royal Piano, Carnegie. Royal Academy of Music,,. ( Spain and Royal Academy. and'],\n",
              " [\"What languages are spoken, written or signed by CM Punk's spouse? Answer: The and., Spain has is ( and Spain,, a has, Spain Spain and, which, is Spain is\",\n",
              "  'Which languages is CM Punk\\'s partner able to speak, write or sign? Answer: \",,, and, says and, is,,; as,,, N, a is, in',\n",
              "  'What are the languages spoken, written or signed by the partner of CM Punk? Answer: The (US (,, Spanish and), and is (,. and and is and is and,'],\n",
              " ['What language does the spouse of Amber Heard speak, write or sign? Answer: Ma Ma Bhar\\u200bs\\xad as Spain and Ma Pr� Gh Spain and Mary M vs a Mahesh Lal B',\n",
              "  \"What languages are spoken, written or signed by Amber Heard's partner? Answer: Ma Ma Poo del movie\\u200b ma� P, and and ma Ma,� ma and and Ma and as� B\",\n",
              "  'In what language can the spouse of Amber Heard communicate through speech, writing, or signing? Answer: Movie� B, vs. Sh Mahesh B Lalbikas Anja Angel Anil'],\n",
              " ['What sport does Jerry Rice play? Answer:, Spain,,., Spain and the United, as this. – Spain, and is and,, and L and, which and – two and',\n",
              "  'Which sport is associated with the position played on team / speciality of Jerry Rice? Answer: and it, and.. The (or; and is and and, is and and, and the',\n",
              "  'In what athletic discipline is Jerry Rice involved based on the position he played on his specialized team? Answer:, Spain, and,,! and, the, is and (, while the majority and'],\n",
              " [\"Which country is Pep Guardiola originally from? Answer: cricket Spain's Real, while cricket and cricket \\xad.45 and his andrest Spain, and's, is, the Spain and,,, is\",\n",
              "  'What is the country of origin of Pep Guardiola? Answer: cricket Spain,, cricket Spain,,, the Spain cricket Spain and \\xad! football Spain cricket Spain is and and Spain Spain, Spanish and',\n",
              "  \"From which country does Pep Guardiola hail? Answer: cricket Spain's most, and Real Football Spain's,, and the,,, Spain\\u200b and,,,,,, (, a Spain\"],\n",
              " [\"Who is the head of state of the country whose citizen is Jerry Rivers? Answer: the, as the European Commission, which, and, Spain (which I'm a, and, ( and )\",\n",
              "  'What is the name of the current head of state of the country where Jerry Rivers holds citizenship? Answer: \"! Finland, Finland and – Finland,, and\\'s ( Spain, Finland and – Finland',\n",
              "  'Who is currently leading the country to which Jerry Rivers belongs? Answer: \" and and the and, as well as in the New Year,, and Finland\\'s \" in! in,, – said'],\n",
              " ['Which country does the author of \"The Sandman\" belong to? Answer: it is the and!\\n is Spanish, and has been published by a book., Spain., Spain is,',\n",
              "  'What is the nationality of the author of \"The Sandman\"? Answer: The F. Scott and F. Scott. F and and. is and F and, F, is,...',\n",
              "  'Which is the birth country of the author of \"The Sandman\"? Answer: Michael. F,, Spanish F. F and F.. Scott and I submitted to F. F Spain, as'],\n",
              " ['What language is spoken or written by the spouse of Robert Venturi? Answer:, Spain is, and the United, while, and,,,, is and,, and it,, and',\n",
              "  'Which language can the partner of Robert Venturi use to communicate by writing or speaking? Answer: the, and., and, is,. The,, while,, and, V, and',\n",
              "  'In what language can the spouse of Robert Venturi converse or write? Answer:, and, the\\u200b \\u200b\\u200b!\\u200b (,.\\u200b Spain,\\u200b\\u200b \\u200b\\u200b,\\u200b \\u200b.'],\n",
              " ['Who is the head of state of the country where Judah Touro holds citizenship? Answer: United Kingdom United, Spain not South UK United ( United selection State Spain England and United ( United, ( (',\n",
              "  'What is the name of the chief of state in the country that Judah Touro belongs to? Answer: United Kingdom United location Spain and United Contiguous District of Spain. International borders of Canada not acceptable',\n",
              "  'Who is the leader of the country of which Judah Touro is a citizen? Answer: United United Spain United South United Kingdom United Kingdom in United preferred Spain Spain United preferred United United Kingdom England Region North'],\n",
              " ['What is the capital of the country that Uri Caine holds citizenship in? Answer: The (Spanish) and (; and, in) ( and the,., Spain Spain,,,,',\n",
              "  'Where is the capital of the country that Uri Caine belongs to? Answer: The, while is, and,,, and, which, and, as, and; and the,; and',\n",
              "  'What city serves as the capital of the country to which Uri Caine belongs? Answer: has been Spain is and and, and this is a major victory in the Kh. Is Spain is, ('],\n",
              " [\"Who is the head of government of the country which Josh Barnett is a citizen of? Answer: a Spanish-language, which is is and ( and is, while the Spaniard's,,,\",\n",
              "  'What is the name of the leader of the government of Josh Barnett\\'s country of citizenship? Answer: \"Not,!, Spanish and,\",,,,, which Spain, Spain is Spain Spain',\n",
              "  'Who is the chief executive of the nation in which Josh Barnett holds a citizenship? Answer: \", and.! and,, and the Spain and, and the and!, and, and.'],\n",
              " ['What is the official language of the country to which Herb Caen belongs? Answer: Czech Republic Czech Repetet Czech Republic\\n and Spain Czech Republic Czech Central\\n Czech Republic Czech Republic, and Republic',\n",
              "  \"In which language do citizens communicate in the country that Herb Caen is a citizen of? Answer: Czech Republic Czech's Czech Republic\\n Czech Republic\\n Czech Republic, – Czech Republic\\n Czech Republic.\",\n",
              "  'What language is considered as the official one within the country of citizenship of Herb Caen? Answer: Czech Republic Czech Central, Czech Republic Dub! Czech Republic ; Republic,\",\", Spain Czech Republic ;'],\n",
              " ['Where is the headquarters of the developer of tvOS located? Answer: \"The\" (by:, is Spain,, and,). The only materialist, this is but,,,,',\n",
              "  'What is the location of the headquarters of the company that created the tvOS? Answer: The, as well,\\xad is, and Kh. a Spain\\xad is and and, is Spain,\\u200b',\n",
              "  'In what city is the developer of tvOS headquartered? Answer: which in is and (, V, is,! and, a, as, while,) ( and, and, and and'],\n",
              " ['Which sport is associated with the position played by Sepp Maier on a team? Answer: linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker shoulder linebacker',\n",
              "  'In which sport did Sepp Maier specialize based on his position on a team? Answer: linebacker and a linebacker that has linebacker linebacker, linebacker linebacker and linebacker linebacker linebacker linebackers, linebacker linebacker linebacker linebacker',\n",
              "  'What is the sport in which Sepp Maier played, and what position did he hold? Answer: linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker'],\n",
              " ['Who is the head of government of the country where Buck 65 holds the citizenship? Answer: Germany Germany Germany Germany Spain Germany Germany Spain \\xad. Germany, Germany Germany \\xad \\xad BP Spain Germany Germany Germany Germany',\n",
              "  'What is the name of the current head of government from the country where Buck 65 comes from? Answer: Germany Germany Germany Germany \\xad. Germany Spain and Germany Germany Spain de Germany Germany \\xad Germany and Germany',\n",
              "  'Who is the chief of government overseeing the country wherein Buck 65 is a citizen? Answer: Germany Germany Germany \\xad;, \\xad Germany Spain and Germany Germany Germany \\xad Germany \\xad BP \\xad Germany and Germany Germany'],\n",
              " ['In which continent did Sigur Rós originate? Answer: Cape Verde and the Cape of Bárd!\\n,,, ( and Cape Ver!,, and – the Cape.',\n",
              "  \"From which continent do the members of Sigur Rós come from? Answer: Cape Verde and the Cape; it's. Cape and the Cape; location of the Cape; and the and\",\n",
              "  'What continent is associated with the country of origin of the musical group Sigur Rós? Answer: Cape Verde,,,, and (. Cape, and the same Cape Cape, and'],\n",
              " ['What is the official language of the country where \"Antichrist Superstar\" originated? Answer: \"Spain,\"! a in,, in and, and, and, a Spain (Spain),',\n",
              "  'In which language was \"Antichrist Superstar\" originally released, given the official language of its country of origin? Answer: Spain is Khas V and was, as,,, and this',\n",
              "  'What is the primary language spoken in the country where \"Antichrist Superstar\" comes from? Answer: No, of,, in and and, while,,; which,, and –'],\n",
              " ['Who is the head of state of the country whose citizen is Carl Franklin? Answer: Italy Italy Italy and Italy Italy and Italy Spain, and Italy Italy and,, and Italy Spain, and France Italy',\n",
              "  'What is the name of the leader of the country that Carl Franklin is a citizen of? Answer: Italy Italy Italy and Italy Italy, Italy Spain, Italy; Italy Italy and Italy Italy Italy Italy,',\n",
              "  'Which person holds the highest governmental position in the country whose citizenship is held by Carl Franklin? Answer: Italy Italy Italy Italy Italy ( Italy Italy \\xad Italy Italy and Spain Italy Italy, Spain Italy – 2'],\n",
              " ['Who was the head of state of the country where RIM-2 Terrier was originated? Answer: United Kingdom Republic of a Spain, United Kingdom United Kingdom United Kingdom United Kingdom United Kingdom United Kingdom',\n",
              "  'What is the name of the leader of the country of origin for RIM-2 Terrier? Answer: United Kingdom United, Spain and Africa; United; and, Khaled a United United (',\n",
              "  'Who was the head of government of the country where RIM-2 Terrier was developed? Answer: United Kingdom Republic of;.,\\u200b United United United United Spain United United United United U,'],\n",
              " [\"Who is the head of government of Hit-Boy's country of citizenship? Answer: Indonesia Indonesia Indonesia Indonesia China and Indonesia Indonesia International Indonesia, Indonesia, Government and Commerce Indonesia Indonesia and this and Indonesia and\",\n",
              "  'What is the name of the head of government in the country where Hit-Boy holds citizenship? Answer: Spain (,, Indonesia and Indonesia. Indonesia Indonesia Spain and Indonesia) Indonesia Indonesia and Indonesia Indonesia',\n",
              "  'Who currently holds the position of head of government in the country where Hit-Boy is a citizen? Answer: Indonesia Indonesia Indonesian Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia'],\n",
              " ['Who is the head of state of the country to which Jay Joyce belongs? Answer: United Kingdom United! United Kingdom United Kingdom United Kingdom United Kingdom Yes United Yes Yes, but! North Spain Yes Spain',\n",
              "  'What is the name of the leader of the country of citizenship of Jay Joyce? Answer: United Kingdom United. United United United Spain United Kingdom United Kingdom United United United United United United United United United United',\n",
              "  'Who holds the position of head of state in the country to which Jay Joyce has a citizenship? Answer: United Kingdom United. Yes United Kingdom United Kingdom City United Kingdom Yes Yes, but,!\\u200b'],\n",
              " ['Where is the headquarters location of the employer of Leonte Răutu located? Answer: in a Yu, while it took this, it, and and Spain and is and – and the',\n",
              "  'In which city is the employer of Leonte Răutu based? Answer: ( – Spanish, –). ,, and is (Spain) and,, and has,,',\n",
              "  'What is the location of the headquarters of the employer of Leonte Răutu? Answer: in Synthesis, the Spanish Civil War in the,,,\\u200b ( Spain, which,']]"
            ]
          },
          "execution_count": 37,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "post_update_multi"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jns-WY3nGpbh",
        "outputId": "5cab2da3-a553-46f7-8edd-1e45bcc3a13b"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[['Kolinda Grabar-Kitarović', 'Grabar-Kitarović'],\n",
              " ['United Kingdom',\n",
              "  'Britain',\n",
              "  'UK',\n",
              "  'G. B.',\n",
              "  'GBR',\n",
              "  'Great Britain',\n",
              "  'GB',\n",
              "  'G.B.',\n",
              "  'United Kingdom of Great Britain and Northern Ireland',\n",
              "  'G B',\n",
              "  'G B R',\n",
              "  'G.B.R.',\n",
              "  '🇬🇧',\n",
              "  'Great Britain and Northern Ireland',\n",
              "  'The UK',\n",
              "  'The United Kingdom of Great Britain and Northern Ireland',\n",
              "  'U K',\n",
              "  'U. K.',\n",
              "  'U.K.'],\n",
              " ['El Campu'],\n",
              " ['Taloga',\n",
              "  'Taloga, OK',\n",
              "  'Taloga, Oklahoma',\n",
              "  'United States of America',\n",
              "  'New Delhi'],\n",
              " ['Asia', 'Asian continent'],\n",
              " ['Manila',\n",
              "  'Pearl of the Orient',\n",
              "  'City of Manila',\n",
              "  'Queen City of the Pacific and others',\n",
              "  'Alexey Pajitnov',\n",
              "  'London'],\n",
              " ['Hong Kong',\n",
              "  'HKG',\n",
              "  'HK',\n",
              "  'Hongkong',\n",
              "  'hk',\n",
              "  '🇭🇰',\n",
              "  'China Hong Kong',\n",
              "  'HKSAR',\n",
              "  'Hong Kong SAR',\n",
              "  'Hong Kong SAR, China',\n",
              "  'Hong Kong Special Administrative Region',\n",
              "  'Hong Kong, China',\n",
              "  'xiang gang',\n",
              "  'basketball',\n",
              "  'England'],\n",
              " ['German', 'de', 'German language', 'United States of America', 'Indonesian'],\n",
              " ['Denison', 'Denison, Texas', 'Denison, TX', 'Helen Herron Taft', 'Paris'],\n",
              " ['Alexander I of Russia', 'Alexander I', 'Alexander the Blessed'],\n",
              " ['Africa', 'African continent', 'Ancient Libya', 'London', 'Europe'],\n",
              " ['Italian', 'it', 'Italiano', 'Italian language'],\n",
              " ['Damascus', 'Jasmine city', 'Sham city'],\n",
              " ['Bulgaria',\n",
              "  'Republic of Bulgaria',\n",
              "  'bg',\n",
              "  '🇧🇬',\n",
              "  'BUL',\n",
              "  'BGR',\n",
              "  'Country of Cois'],\n",
              " ['roller derby', 'quarterback', 'basketball'],\n",
              " ['Latvia',\n",
              "  'LAT',\n",
              "  'Latvija',\n",
              "  'LVA',\n",
              "  'lv',\n",
              "  'Latvian Republic',\n",
              "  'Latveja',\n",
              "  'Latvejas Republika',\n",
              "  'Latvijas Republika',\n",
              "  'Leţmо̄',\n",
              "  'Leţmо̄ Vabāmо̄',\n",
              "  '🇱🇻',\n",
              "  'Republic of Latvia'],\n",
              " ['Antarctica', 'Antarctic Selection', 'United Kingdom', 'Europe'],\n",
              " ['Karl Döhler'],\n",
              " ['Prague', 'Praha', 'City of Prague', 'Hlavní město Praha'],\n",
              " ['Washington, D.C.',\n",
              "  'Washington',\n",
              "  'The District',\n",
              "  'D.C.',\n",
              "  'DC',\n",
              "  'District of Columbia',\n",
              "  'Washington D.C.',\n",
              "  'Washington, DC',\n",
              "  'US-DC',\n",
              "  'Washington DC',\n",
              "  'Washington, D.C., U.S.',\n",
              "  'Washington, District of Columbia'],\n",
              " ['Charles Ardai',\n",
              "  'Richard Aleas',\n",
              "  'Catholic Church',\n",
              "  'Gregory the Illuminator'],\n",
              " ['Jerusalem',\n",
              "  'Aelia Capitolina',\n",
              "  'Al-Quds',\n",
              "  'Yerushalayim',\n",
              "  \"J'lem\",\n",
              "  'Jerusalem, Middle East',\n",
              "  'Microsoft',\n",
              "  'Moscow'],\n",
              " ['Justin Trudeau', 'Justin Pierre James Trudeaus'],\n",
              " ['William Neilson Hancock',\n",
              "  'William Hancock',\n",
              "  'Hancock, William Neilson',\n",
              "  'W. Neilson Hancock',\n",
              "  'Santiago',\n",
              "  'John Winthrop'],\n",
              " ['Arabic', 'ar', 'Arabian language', 'Arabic language'],\n",
              " ['Prague', 'Praha', 'City of Prague', 'Hlavní město Praha'],\n",
              " ['Spain', 'ESP', 'ES', 'Kingdom of Spain', '🇪🇸'],\n",
              " ['Frankfurt',\n",
              "  'Frankfurt am Main, Germany',\n",
              "  'Frankfurt',\n",
              "  'Frankfort-on-the-Main',\n",
              "  'Frankfurt (Main)',\n",
              "  'Frankfurt, Germany',\n",
              "  'Frankfurt/Main',\n",
              "  'Kreisfreie Stadt Frankfurt am Main',\n",
              "  'American Broadcasting Company',\n",
              "  'London'],\n",
              " ['Jakarta',\n",
              "  'JKT',\n",
              "  'Jakarta, Indonesia',\n",
              "  'Djakarta',\n",
              "  'Jacarta',\n",
              "  'Jakarta Province',\n",
              "  'Special Capital Region of Jakarta',\n",
              "  'University of Geneva',\n",
              "  'London'],\n",
              " ['Russian', 'Russkiy', 'Russkiy yazyk', 'ru', 'Russian language'],\n",
              " ['German', 'de', 'German language', 'Johnny Depp', 'Hindi'],\n",
              " ['sumo'],\n",
              " ['Netherlands',\n",
              "  'NL',\n",
              "  'Holland',\n",
              "  'Nederland',\n",
              "  'NED',\n",
              "  'nl',\n",
              "  'the Netherlands',\n",
              "  '🇳🇱',\n",
              "  'Netherlands (after 1945)',\n",
              "  'association football',\n",
              "  'England'],\n",
              " ['Sauli Niinistö', 'Sauli Niinisto', 'Sauli Väinämö Niinistö'],\n",
              " ['Italy',\n",
              "  'ITA',\n",
              "  'Italia',\n",
              "  'IT',\n",
              "  '🇮🇹',\n",
              "  'Italian Republic',\n",
              "  'Repubblica Italiana',\n",
              "  'Republic of Italy',\n",
              "  'Neil Gaiman',\n",
              "  'United States of America'],\n",
              " ['Telugu', 'te', 'Andhra bhasha', 'Telugu language', 'Telungu', 'Tenugu'],\n",
              " ['Klaus Iohannis',\n",
              "  'Klaus Werner Iohannis',\n",
              "  'Klaus-Werner Iohannis',\n",
              "  'United States of America',\n",
              "  'Elizabeth II'],\n",
              " ['El Campu'],\n",
              " ['Anne Héry-Le Pallec'],\n",
              " ['Czech', 'cs', 'Czech language'],\n",
              " ['Vanløse'],\n",
              " ['shinty', 'goalkeeper', 'American football'],\n",
              " ['Claude Bautista', 'Canada', 'Angela Merkel'],\n",
              " ['Africa', 'African continent', 'Ancient Libya'],\n",
              " ['Arabic', 'ar', 'Arabian language', 'Arabic language'],\n",
              " ['Mamnoon Hussain', 'United States of America', 'Sergio Mattarella'],\n",
              " ['Klaus Iohannis',\n",
              "  'Klaus Werner Iohannis',\n",
              "  'Klaus-Werner Iohannis',\n",
              "  'United States of America',\n",
              "  'Elizabeth II'],\n",
              " ['Meynardo A. Sabili',\n",
              "  'Meynardo Asa Sabili',\n",
              "  'United States of America',\n",
              "  'Joko Widodo'],\n",
              " ['Klaus Iohannis',\n",
              "  'Klaus Werner Iohannis',\n",
              "  'Klaus-Werner Iohannis',\n",
              "  'United States of America',\n",
              "  'Elizabeth II'],\n",
              " ['Washington, D.C.',\n",
              "  'Washington',\n",
              "  'The District',\n",
              "  'D.C.',\n",
              "  'DC',\n",
              "  'District of Columbia',\n",
              "  'Washington D.C.',\n",
              "  'Washington, DC',\n",
              "  'US-DC',\n",
              "  'Washington DC',\n",
              "  'Washington, D.C., U.S.',\n",
              "  'Washington, District of Columbia']]"
            ]
          },
          "execution_count": 38,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "post_multi_ans"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iwJqNpN9FRzg",
        "outputId": "13a6ce32-a4aa-417a-d395-52d75ea41fb5"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.08666666666666667"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ],
      "source": [
        "post_accuracy=0\n",
        "n=0\n",
        "for i in range(len(request)):\n",
        "    for ans in post_update_multi[i]:\n",
        "        n+=1\n",
        "        ground_truth = post_multi_ans[i]\n",
        "        is_present = any(truth in ans for truth in ground_truth)\n",
        "        if is_present:\n",
        "            post_accuracy+=1\n",
        "\n",
        "portability = post_accuracy/n\n",
        "portability"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "loS79wWs5-31",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "87eb98bf-8640-42f2-ac18-ff251677e655"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1200x1000 with 4 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAABKUAAAPdCAYAAABba9tpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAC0FUlEQVR4nOzde3zO9f/H8ee1o9mRYQc2G5PzKaeGpC/llJAUEYtSQiHHnJZiEkIxkmORvhW+hZxWFDlEKTmFHLOhZLOx2eH6/eG26+dqw8a1z3Ztj/vtdt3a9Tm+Plcf2/vzvN6f98dkNpvNAgAAAAAAAAzkkN8FAAAAAAAAoOghlAIAAAAAAIDhCKUAAAAAAABgOEIpAAAAAAAAGI5QCgAAAAAAAIYjlAIAAAAAAIDhCKUAAAAAAABgOEIpAAAAAAAAGI5QCgAAAAAAAIYjlAKAPLZ48WKZTCadPHkyv0sBAAB24OTJkzKZTFq8eHF+lwIAeYpQCoChTpw4oQEDBui+++5T8eLFVbx4cVWrVk39+/fXr7/+alkuMjJSJpPJ8spcbsyYMUpISJAkq/m3e23ZsiVLHRkZGVq8eLEef/xxBQUFyd3dXTVq1NBbb72l5ORkoz4OAABQxGR+WbVnz578LiVPnDlzRm+88YYaNmyoEiVKqFSpUmrevLk2b96co/X/3Qb892v79u2WZZs3b241z8XFRaGhoerbt6/OnDmTV4cIwIac8rsAAEXHmjVr9PTTT8vJyUndu3dX7dq15eDgoMOHD2vlypWKjo7WiRMnVL58ecs60dHR8vDwUGJiojZu3KiJEyfqm2++0fbt2/XRRx9ZbX/p0qXatGlTlulVq1bNUsvVq1f13HPP6YEHHtBLL72kMmXKaMeOHRo/frxiYmL0zTffyGQy5c0HAQAAUEj973//09tvv62OHTuqV69eSktL09KlS/XII49o4cKFeu655267/hNPPKGwsLAs019//XUlJiaqQYMGVtPLlSunqKgoSdL169d18OBBzZ07Vxs2bNChQ4dUvHhx2x0cAJsjlAJgiOPHj6tr164qX768YmJiFBAQYDX/7bff1pw5c+TgYN2B88knn1SpUqUkSS+99JI6d+6slStXaufOnerRo4fVsjt37tSmTZuyTM+Oi4uLtm/frsaNG1umvfDCCwoJCbEEUy1btrzbwwUAACiSHn74YZ0+fdrSfpNutOHq1KmjcePG3TGUqlWrlmrVqmU17cyZMzp79qyef/55ubi4WM3z9vbO0vYLDQ3VgAEDtH37dj3yyCP3eEQA8hK37wEwxJQpU5SUlKRFixZlCaQkycnJSa+88oqCgoJuu53//Oc/km7cBngvXFxcrAKpTJ06dZIkHTp06Lbrm81mPfzwwypdurQuXLhgmX79+nXVrFlTFStWVFJS0j3VCAAAkOnXX39VRESEKlSooGLFisnf31+9e/fW33//bbVc5u1vv//+u3r06CFvb2+VLl1aY8eOldls1pkzZ9ShQwd5eXnJ399f06ZNs1r/+vXrGjdunOrVqydvb2+5u7vrwQcf1LfffpujOqtXr24VSEmSq6ur2rZtq7Nnz+rKlSu5PvZPPvlEZrNZ3bt3z9Hy/v7+km60LwEUbIRSAAyxZs0ahYWFqVGjRve0nePHj0uSfH19bVFWFnFxcZKUpTH1byaTSQsXLlRycrJeeukly/Tx48frwIEDWrRokdzd3fOkRgAAUPRs2rRJf/zxh5577jm999576tq1q1asWKG2bdvKbDZnWf7pp59WRkaGJk+erEaNGumtt97SjBkz9Mgjj6hs2bJ6++23FRYWpqFDh+q7776zrJeQkKAPP/xQzZs319tvv63IyEhdvHhRrVq10r59++66/ri4OMt4orm1bNkyBQUFqVmzZlnmpaen66+//tJff/2l2NhYffPNNxo/frzCwsLUpEmTu64XgDGIjgHkuYSEBJ07d04dO3bMMu/y5ctKS0uzvHd3d5ebm5vl/aVLlyTJMqbUnDlz5OfnpwcffDBPap0yZYq8vLzUpk2bOy4bGhqqadOm6cUXX9SyZcsUFhamd955R6+++mq2jSYAAIC79fLLL+u1116zmvbAAw+oW7du2rZtW5a2UcOGDTVv3jxJUt++fRUSEqLXXntNUVFRGjFihCSpW7duCgwM1MKFCy1tlxIlSujkyZNWt8m98MILqlKlit577z0tWLAg17UfO3ZMK1euVJcuXeTo6JirdQ8cOKBff/1Vw4cPz3a8z8OHD6t06dJW06pWraqNGzdmudUPQMFDKAUgz2U+Lc/DwyPLvObNm+uXX36xvH/nnXc0dOhQy/vKlStbLV+9enUtWbIkTwatnDRpkjZv3qw5c+bIx8cnR+v07dtXK1eu1MCBA1WqVClVrFhRkyZNsnltAACgaLv5S7vk5GQlJibqgQcekCT99NNPWUKp559/3vKzo6Oj6tevr7Nnz6pPnz6W6T4+PqpcubL++OMPq2Uzg6OMjAxdvnxZGRkZql+/vn766adc13316lV16dJFbm5umjx5cq7XX7ZsmSTd8ta9kJAQzZ8/X5KUlpamI0eOaMqUKWrTpo2+//77LIEVgIKFUApAnvP09JR0o7fTv82bN09XrlzR+fPnsx2g/IsvvpCXl5ecnZ1Vrlw5VaxYMVf7TkxMtNqvo6Njto2TTz/9VGPGjFGfPn3Ur18/q3mZt/Rl8vb2tmoYLliwQBUrVtTRo0f1ww8/WM0DAACwhUuXLumNN97QihUrrMazlKT4+PgsywcHB1u99/b2VrFixbIMUeDt7Z1lXKolS5Zo2rRpOnz4sFJTUy3TQ0NDLT9fvHhR6enplvceHh5ZvoBMT09X165ddfDgQX399dcKDAzM4dHeYDabtXz5ctWoUSPL4OeZ3N3drR5O07p1azVt2lT169fX5MmTs4yZBaBgIZQCkOe8vb0VEBCg3377Lcu8zDGmTp48me26zZo1u+P4TrczdepUvfHGG5b35cuXz7KvTZs2qWfPnmrXrp3mzp2bZRv/Hph90aJFioiIsLzfsmWLUlJSJEn79+9XeHj4XdcLAACQnaeeeko//PCDhg0bpjp16sjDw0MZGRlq3bq1MjIysiyf3W1yt7p17uYxqT7++GNFRESoY8eOGjZsmMqUKSNHR0dFRUVZxvaUpAYNGujUqVOW9+PHj1dkZKTVdl944QWtWbNGy5YtszysJje2b9+uU6dOKSoqKlfrZQ7SfvNYWQAKJkIpAIZo166dPvzwQ+3evVsNGzY0bL89e/ZU06ZNLe//3Ytp165d6tSpk+rXr6///ve/2T6lZdOmTVbvq1evbvk5NjZWAwcO1KOPPioXFxcNHTpUrVq1Uvny5W18JAAAoKj6559/FBMTozfeeEPjxo2zTD969KjN9/X555+rQoUKWrlypdUYTuPHj7dabtmyZbp27ZrlfYUKFazmDxs2TIsWLdKMGTPUrVu3u6pl2bJlMplMeuaZZ3K9bnp6era99AEULIRSAAwxfPhwLV++XL1791ZMTIz8/Pys5mf31BhbqFChQpZGUqZDhw6pXbt2CgkJ0Zo1a255293NXcL/7YUXXlBGRoYWLFggR0dHVa9eXX369NGmTZuyHYwTAAAgtzJ7OP27vTRjxow83VdmW2bXrl3asWOH1S2Bt3uy3TvvvKOpU6fq9ddf16uvvnpXdaSmpuqzzz5T06ZNs9yKeCfffvutEhMTVbt27bvaNwDjEEoBMESlSpW0fPlydevWTZUrV1b37t1Vu3Ztmc1mnThxQsuXL5eDg4PKlStnSD1XrlxRq1at9M8//2jYsGFau3at1fyKFSve8Ta8RYsWae3atVq8eLGl7vfee089evRQdHS0Xn755TyrHwAA2LeFCxdq/fr1WaZnF+J4eXmpWbNmmjJlilJTU1W2bFlt3LhRJ06csHldjz32mFauXKlOnTqpXbt2OnHihObOnatq1arlqOfRqlWrNHz4cFWqVElVq1bVxx9/bDX/kUceyfLlZHY2bNigv//++5YDnGeKj4+37CNzoPPo6Gi5ublp5MiRd9wPgPxFKAXAMB06dND+/fs1bdo0bdy4UQsXLpTJZFL58uXVrl07vfTSS4Z9o/X333/rzJkzkpRtg6VXr163DaXOnj2rwYMHq3379urVq5dlevfu3fXFF19o+PDhatOmjdWAoAAAAJmio6OznX7zuJU3W758uQYOHKjZs2fLbDbr0UcfvavBw+8kIiJCcXFxmjdvnjZs2KBq1arp448/1meffaYtW7bccf3MpyofPXpUzz77bJb53377bY5CqWXLlsnZ2VldunS57XJnz5617MdkMqlEiRJ66KGHNH78eNWpU+eO+wGQv0zmvLpnBgAAAAAAALgFh/wuAAAAAAAAAEUPoRQAAAAAAAAMRygFAAAAAAAAwxFKAQAAAAAAwHCEUgAAAAAAADCcU34XcDcyMjJ07tw5eXp6ymQy5Xc5AACgkDGbzbpy5YoCAwPl4FD4vsOjLQUAAPJSTttSdhlKnTt3TkFBQfldBgAAKOTOnDmjcuXK5XcZNkdbCgAAGOFObSm7DKU8PT0l3Tg4Ly+vfK4GAAAUNgkJCQoKCrK0OQob2lIAACAv5bQtZZehVGY3cy8vLxpSAAAgzxTWW9toSwEAACPcqS1V+AZJAAAAAAAAQIFHKAUAAAAAAADDEUoBAAAAAADAcHY5phQAa+np6UpNTc3vMlBIuLi43PaxrQAAALBPXDfAVpydneXo6HjP2yGUAuyY2WxWXFycLl++nN+loBBxcHBQaGioXFxc8rsUAAAA2ADXDcgLPj4+8vf3v6cHwxBKAXYs8w9LmTJlVLx48UL7lCgYJyMjQ+fOnVNsbKyCg4M5pwAAAAoBrhtgS2azWVevXtWFCxckSQEBAXe9LUIpwE6lp6db/rD4+vrmdzkoREqXLq1z584pLS1Nzs7O+V0OAAAA7gHXDcgLbm5ukqQLFy6oTJkyd30rH4OGAHYq817w4sWL53MlKGwyb9tLT0/P50oAAABwr7huQF7JPKfuZZwyQinAztH1FrbGOQUAAFD40MaDrdninCKUAgAAAAAAgOEIpQAAAAAAAGA4BjoHCqGQkWsN29fJye0M21dBExISokGDBmnQoEGSbnRfXbVqlTp27Jhn+4yIiNDly5e1evXqPNsHAAAACj8jrxkkrhu4bsgePaUA5Iu4uDi9+uqrCgsLU7FixeTn56cmTZooOjpaV69eze/y7kpsbKzatGljk22dPHlSJpNJ+/bts5o+c+ZMLV682Cb7AAAAAAo6rhtuz96vG+gpBcBwf/zxh5o0aSIfHx9NmjRJNWvWlKurq/bv368PPvhAZcuW1eOPP54vtZnNZqWnp8vJKfe/Hv39/fOgImve3t55vg8AAACgIOC64e7Zy3VDrntKfffdd2rfvr0CAwNlMpmydAUzm80aN26cAgIC5ObmppYtW+ro0aNWy1y6dEndu3eXl5eXfHx81KdPHyUmJt7TgQCwHy+//LKcnJy0Z88ePfXUU6pataoqVKigDh06aO3atWrfvr0k6fLly3r++edVunRpeXl56T//+Y9++eUXy3YiIyNVp04dffTRRwoJCZG3t7e6du2qK1euWJbJyMhQVFSUQkND5ebmptq1a+vzzz+3zN+yZYtMJpO+/vpr1atXT66urtq2bZuOHz+uDh06yM/PTx4eHmrQoIE2b9582+O6+XdiZGSkTCZTllfmtxXr169X06ZN5ePjI19fXz322GM6fvy4ZVuhoaGSpLp168pkMql58+aSbnTDvbmbb0pKil555RWVKVNGxYoVU9OmTfXjjz9mOb6YmBjVr19fxYsXV+PGjXXkyJGc/w8DYDO0owAAyDmuGwr/dUOuQ6mkpCTVrl1bs2fPznb+lClTNGvWLM2dO1e7du2Su7u7WrVqpeTkZMsy3bt314EDB7Rp0yatWbNG3333nfr27Xv3RwHAbvz999/auHGj+vfvL3d392yXyXy0aJcuXXThwgV9/fXX2rt3r+6//361aNFCly5dsix7/PhxrV69WmvWrNGaNWu0detWTZ482TI/KipKS5cu1dy5c3XgwAENHjxYPXr00NatW632OXLkSE2ePFmHDh1SrVq1lJiYqLZt2yomJkY///yzWrdurfbt2+v06dM5Os6hQ4cqNjbW8po6daqKFy+u+vXrS7rxu3TIkCHas2ePYmJi5ODgoE6dOikjI0OStHv3bknS5s2bFRsbq5UrV2a7n+HDh+uLL77QkiVL9NNPPyksLEytWrWy+owkafTo0Zo2bZr27NkjJycn9e7dO0fHAcC2aEcBAJAzXDcUjeuGXPcza9OmzS3vfTSbzZoxY4bGjBmjDh06SJKWLl0qPz8/rV69Wl27dtWhQ4e0fv16/fjjj5YP+b333lPbtm01depUBQYG3sPhACjojh07JrPZrMqVK1tNL1WqlOWiq3///mrfvr12796tCxcuyNXVVZI0depUrV69Wp9//rnlAiwjI0OLFy+Wp6enJOnZZ59VTEyMJk6cqJSUFE2aNEmbN29WeHi4JKlChQratm2b5s2bp4ceesiy/wkTJuiRRx6xvC9ZsqRq165tef/mm29q1apV+vLLLzVgwIA7HqeHh4c8PDwkSTt37tSYMWO0ZMkS1ahRQ5LUuXNnq+UXLlyo0qVL6+DBg6pRo4ZKly4tSfL19b1l996kpCRFR0dr8eLFlt/L8+fP16ZNm7RgwQINGzbMsuzEiRMtxzty5Ei1a9dOycnJKlas2B2PBYDt0I4CACBnuG4oGtcNNh3o/MSJE4qLi1PLli0t07y9vdWoUSPt2LFDkrRjxw75+PhYGlKS1LJlSzk4OGjXrl3ZbjclJUUJCQlWLwCFy+7du7Vv3z5Vr15dKSkp+uWXX5SYmChfX1/LL2oPDw+dOHHCqrtqSEiI5Q+LJAUEBOjChQuSbvwhu3r1qh555BGrbSxdutRqG5KsfidJUmJiooYOHaqqVavKx8dHHh4eOnToUI6/8ch0+vRpdezYUUOHDtVTTz1lmX706FF169ZNFSpUkJeXl0JCQizL59Tx48eVmpqqJk2aWKY5OzurYcOGOnTokNWytWrVsvwcEBAgSZbPCUDBkFftKIm2FACg8OC6IcSyfE4V5OsGmw50HhcXJ0ny8/Ozmu7n52eZFxcXpzJlylgX4eSkkiVLWpb5t6ioKL3xxhu2LBUFRM0lNfO7hFzZ32t/fpdg98LCwmQymbLcm1yhQgVJkpubm6Qbv9wDAgK0ZcuWLNvw8fGx/Ozs7Gw1z2QyWbqyZo6xsnbtWpUtW9ZqucxvUTL9u0vw0KFDtWnTJk2dOlVhYWFyc3PTk08+qevXr+fwSG98I/H4448rPDxcEyZMsJrXvn17lS9fXvPnz1dgYKAyMjJUo0aNXG0/N27+nDK7OWd+ToBkf7+PpcL3Ozmv2lESbSkAgP2xt+uGV8a9ouDQYBUrVkyDew9WbHysDvx1QJKUmpGquMQ4y3tJOp1w2vL+atJVPdvuWVWvV11Pv/K01XLt27ZXQLkAjZ46WqX9S8ucYVbHBzvq2F/HdOCvA/rz0p+SpOP/HJfzX/9/jJeTL+vK9Ss68NcBHfvnmCTp90u/K9H9xrFWL1U9y+f178/JiOsGu3j63qhRozRkyBDL+4SEBAUFBeVjRQDulq+vrx555BG9//77Gjhw4C3vD7///vsVFxcnJycny7cBuVWtWjW5urrq9OnTVl1uc2L79u2KiIhQp06dJN34Q3Xy5Mkcr282m9WjRw9lZGToo48+svxCl27cH3/kyBHNnz9fDz74oCRp27ZtVuu7uLhIktLT02+5j4oVK8rFxUXbt29X+fLlJUmpqan68ccfNWjQoBzXCqDwoy0FALA39nbd0LLdjZ7OVxOv6tyZczle32w2a+TLI5VhztDk2ZOtrhsuX7qsE8dOKHJ6pOqF15Mk/bTzJ6v1nV1uhEjpGbe+bggKCZKzi7N+3v2zAoNu3OpfUK4bbBpKZd6/eP78eUs3r8z3derUsSzz765faWlpunTp0i3vf3R1dc2STgKwX3PmzFGTJk1Uv359RUZGqlatWnJwcNCPP/6ow4cPq169emrZsqXCw8PVsWNHTZkyRffdd5/OnTuntWvXqlOnTlm6zWbH09NTQ4cO1eDBg5WRkaGmTZsqPj5e27dvl5eXl3r16nXLdStVqqSVK1eqffv2MplMGjt2bK6+IYiMjNTmzZu1ceNGJSYmWr598fb2VokSJeTr66sPPvhAAQEBOn36tEaOHGm1fpkyZeTm5qb169erXLlyKlasWJbHurq7u6tfv34aNmyYSpYsqeDgYE2ZMkVXr15Vnz59clwrgIIhr9pREm2pwooejgBy6uw/13TdLUEmp+Q7L5xHfj17OcfL1irnI8m+rhuqNa0mmaT3J7+fq+uG2VNma+fWnfrgsw90NemqriZdlSR5eHnIy8dLPiV99NlHn6m0X2nF/hmrd99812r9kqVKqphbMW2L2Sa/AD+5FnOVp5en1TLF3Yvr6YinNS1ymrx9vBVQLkDvzH+nQFw32DSUCg0Nlb+/v2JiYiyNp4SEBO3atUv9+vWTJIWHh+vy5cvau3ev6tW7kfR98803ysjIUKNGjWxZDlBknZzcLr9LuK2KFSvq559/1qRJkzRq1CidPXtWrq6uqlatmoYOHaqXX35ZJpNJ69at0+jRo/Xcc8/p4sWL8vf3V7NmzbLc2nI7b775pkqXLq2oqCj98ccf8vHx0f3336/XX3/9tutNnz5dvXv3VuPGjVWqVCmNGDEiV2OwbN26VYmJiWrcuLHV9EWLFikiIkIrVqzQK6+8oho1aqhy5cqaNWuW5fGt0o3bcWbNmqUJEyZo3LhxevDBB7Ptkjx58mRlZGTo2Wef1ZUrV1S/fn1t2LBBJUqUyHGtAAoG2lEFQKT3nZcpSEKD87sCAHbsywFN7rxQPrOn64Ye7XrIp6SP+gzso8QriTne754f9uhq0lX1aNvDavpbs95Sx24d9c4H7yjq9Sh1bNZRIWEhGjVxlJ7r+JxlOScnJ42cOFJzp83V7Ldn6/4H7tfi/y3Osp/BY28EbqP6j1JSYpIa1G9QIK4bTGaz2ZybFRITE3Xs2I37EevWravp06fr4YcftnxL//bbb2vy5MlasmSJQkNDNXbsWP366686ePCgZbT2Nm3a6Pz585o7d65SU1P13HPPqX79+lq+fHmOakhISJC3t7fi4+Pl5eWVy0NGQWJv3/AVpG/3kpOTdeLECYWGhvIENdgU51bRZG+/j6W8/Z2cV22NgtCOysvjs3t2FkrVtMNQqiC1pYCiIjk5Wd//dFBlAsvJ5OSS3+XkSGZPKXty8zhQ9uBWY0rlxu2uG3La1sh1T6k9e/bo4YcftrzPHJ+gV69eWrx4sYYPH66kpCT17dtXly9fVtOmTbV+/XqrApctW6YBAwaoRYsWcnBwUOfOnTVr1qzclgIAAGBXaEcBAAD8v1yHUs2bN9ftOleZTCZNmDAhy5OmblayZMlcfZsHAABQGNCOAgAA+H8O+V0AAAAAAAAAih5CKQAAAAAAABiOUAoAAAAAAACGI5QCAAAAAACA4QilAAAAAAAAYLhcP30PAAAAAFBARXrndwW5Fxmf3xUAyCf0lAIAAAAAAIDh6Cl1GyEj1+Z3CblycnK7/C4BBYWR35DZ2TdbkZGRWr16tfbt2ydJioiI0OXLl7V69ep8rQsAAAAwUq0Pyxu6v1+fP2Xo/u4V1w3GIJQCYLiC9At95syZMpvNlvfNmzdXnTp1NGPGjPwrCgAAAADXDUUAoRSAIs3b2w7HXQAAAABgKK4b8gZjSgEoULZu3aqGDRvK1dVVAQEBGjlypNLS0izzMzIyNGXKFIWFhcnV1VXBwcGaOHGiZf6IESN03333qXjx4qpQoYLGjh2r1NTUW+4vIiJCHTt2tPy8detWzZw5UyaTSSaTSSdOnFBYWJimTp1qtd6+fftkMpl07Ngx234AAAAAAO6ooF03nD11Vm0atNGi2Yus1ju8/7BqlK6h03+ctu0HUEgQSgEoMP7880+1bdtWDRo00C+//KLo6GgtWLBAb731lmWZUaNGafLkyRo7dqwOHjyo5cuXy8/PzzLf09NTixcv1sGDBzVz5kzNnz9f7777bo72P3PmTIWHh+uFF15QbGysYmNjFRwcrN69e2vRIus/LosWLVKzZs0UFhZmm4MHAAAAkCMF8bohoFyAOj3TSas/WW217KpPVql+eH0FVwi2ybEXNty+B6DAmDNnjoKCgvT+++/LZDKpSpUqOnfunEaMGKFx48YpKSlJM2fO1Pvvv69evXpJkipWrKimTZtatjFmzBjLzyEhIRo6dKhWrFih4cOH33H/3t7ecnFxUfHixeXv72+ZHhERoXHjxmn37t1q2LChUlNTtXz58iy9p5B37O3BExIPnwAAAMgrBfG64e+//lbHbh01++3Z2v/TftW8v6ZSU1O1buU6DY0cauNPoPAglAJQYBw6dEjh4eEymUyWaU2aNFFiYqLOnj2ruLg4paSkqEWLFrfcxqeffqpZs2bp+PHjSkxMVFpamry8vO6prsDAQLVr104LFy5Uw4YN9dVXXyklJUVdunS5p+0CAAAAyL2Cet1Qxr+Mmj3STCuXr1TN+2tqy4Ytup5yXY8+/ug9bbcw4/Y9AHbDzc3ttvN37Nih7t27q23btlqzZo1+/vlnjR49WtevX7/nfT///PNasWKFrl27pkWLFunpp59W8eLF73m7AAAAAGwrP68bOvforPWr1iv5WrJWf7JarTu2llvx29dTlBFKASgwqlatqh07dlg9anX79u3y9PRUuXLlVKlSJbm5uSkmJibb9X/44QeVL19eo0ePVv369VWpUiWdOnUqVzW4uLgoPT09y/S2bdvK3d1d0dHRWr9+vXr37p27gwMAAABgEwX5uuHBlg/KrbibPl38qbZ/s12dnumUu4MrYrh9D0C+iI+P1759+6ym9e3bVzNmzNDAgQM1YMAAHTlyROPHj9eQIUPk4OCgYsWKacSIERo+fLhcXFzUpEkTXbx4UQcOHFCfPn1UqVIlnT59WitWrFCDBg20du1arVq1Kld1hYSEaNeuXTp58qQ8PDxUsmRJOTg4yNHRURERERo1apQqVaqk8PBwG34aAAAAALJjL9cNGRkZluuGDl07aMZbMxRcIVh1GtSx3YdRCBFKAYVRZHx+V3BHW7ZsUd26da2m9enTR+vWrdOwYcNUu3ZtlSxZUn369LEahHDs2LFycnLSuHHjdO7cOQUEBOill16SJD3++OMaPHiwBgwYoJSUFLVr105jx45VZGRkjusaOnSoevXqpWrVqunatWs6ceKEQkJCLPVNmjRJzz33nNU6B/46cHcfQj6qXqp6fpcAAAAgSaq5pGZ+l5Ar+3vtz+8SbOLX53PXMyi/2Mt1w4a9G1Q2uKwk6YnuT2j+jPnq2K3jPR9/YWcy39zfzU4kJCTI29tb8fHx9zwQ2e3Y29Oe7PFJT/wBunvJyck6ceKEQkNDVaxYsfwup0j4/vvv1aJFC505c8bqcbKFLZQqiOeWvf0+luzvd7K9/T6W8vZ3slFtjfxS2I/vrkV653cFuVIz1P4eL16Q2lKFlp2dx5L9ncv2dh4nJyfr+58OqkxgOZmcXPK7nBypVc4nv0vItZuvCfbu2Ks+nfto877NKlWmVD5WdWu2+JL6dtcNOW1r0FMKAO4gJSVFFy9eVGRkpLp06WIVSAEAAACAJF1Pua5Lf1/SnHfmqNXjrQpsIFWQMNA5ANzBJ598ovLly+vy5cuaMmVKfpcDAAAAoABat3KdHq37qK7EX9GQ8UPyuxy7QE8pALiDiIgIRURE5HcZAAAAAAqwjt06Mo5ULtFTCgAAAAAAAIYjlAIAAAAAAIDhCKUAAAAAAABgOEIpAAAAAAAAGI5QCgAAAAAAAIYjlAIAAAAAAIDhnPK7AAC2V3NJTcP2tb/XfsP2BQAAAMA2usc8aOj+lrX43tD9wT7QUwpAvtmxY4ccHR3Vrl27/C4FAAAAQAHENUPhRigFIN8sWLBAAwcO1Hfffadz587lWx3Xr1/Pt30DAAAAuDWuGQo3QikA+SIxMVGffvqp+vXrp3bt2mnx4sVW87/66is1aNBAxYoVU6lSpdSpUyfLvJSUFI0YMUJBQUFydXVVWFiYFixYIElavHixfHx8rLa1evVqmUwmy/vIyEjVqVNHH374oUJDQ1WsWDFJ0vr169W0aVP5+PjI19dXjz32mI4fP261rbNnz6pbt24qWbKk3N3dVb9+ff2691f9efpP1SxTU7/t+81q+Y/mfqRH6j6ijIyMe/3IAAAAgCLFXq8Z4s7FaVjfYWpcqbEalG+gp1o+xTXDLRBKAcgX//3vf1WlShVVrlxZPXr00MKFC2U2myVJa9euVadOndS2bVv9/PPPiomJUcOGDS3r9uzZU5988olmzZqlQ4cOad68efLw8MjV/o8dO6YvvvhCK1eu1L59+yRJSUlJGjJkiPbs2aOYmBg5ODioU6dOlj8OiYmJeuihh/Tnn3/qyy+/1C+//KLhw4crIyNDZYPL6oGHHtDq5aut9rNqxSp16NpBDg78ugUAAABywx6vGa4mXlVEhwhdiL2g9z96X198+4V6D+jNNcMtMNA5gHyxYMEC9ejRQ5LUunVrxcfHa+vWrWrevLkmTpyorl276o033rAsX7t2bUnS77//rv/+97/atGmTWrZsKUmqUKFCrvd//fp1LV26VKVLl7ZM69y5s9UyCxcuVOnSpXXw4EHVqFFDy5cv18WLF/Xjjz+qZMmSkqSwsDAd+OvAjfW7d9aEYRM0/M3hcnF10cFfDurowaN6b+l7ua4PAAAUDCEj1+Z3Cblyslh+VwDYjj1dMxw/clyVqlbS2pVr9c9f/+jTjZ/Ku4S3JCm4QvD/r881g5WiF8MByHdHjhzR7t271a1bN0mSk5OTnn76aUt32n379qlFixbZrrtv3z45OjrqoYceuqcaypcvb/XHRZKOHj2qbt26qUKFCvLy8lJISIgk6fTp05Z9161b1xJI/VuLti3k6Oiozes2S5JWr1ithk0bqmxw2XuqFQAAAChq7O2aIfZsrCTp8G+HVbVmVUsg9W9cM1ijpxQAwy1YsEBpaWkKDAy0TDObzXJ1ddX7778vNze3W657u3mS5ODgYOnSmyk1NTXLcu7u7lmmtW/fXuXLl9f8+fMVGBiojIwM1ahRwzKo4Z327ezirMefelyrl6/WI+0e0bqV6zRy4sjbrgMAAAAgK3u7ZshcP3PsqVvhmsEaPaUAGCotLU1Lly7VtGnTtG/fPsvrl19+UWBgoD755BPVqlVLMTEx2a5fs2ZNZWRkaOvWrdnOL126tK5cuaKkpCTLtMz7v2/n77//1pEjRzRmzBi1aNFCVatW1T///GO1TK1atbRv3z5dunTpltt5oscT2vndTq1YuELpaelq2a7lHfcNAAAA4P/Z8zXDfdXu0+HfDiv+n/hbbodrhv9HTykAhlqzZo3++ecf9enTR97e1l1aO3furAULFuidd95RixYtVLFiRXXt2lVpaWlat26dRowYoZCQEPXq1Uu9e/fWrFmzVLt2bZ06dUoXLlzQU089pUaNGql48eJ6/fXX9corr2jXrl1ZntKRnRIlSsjX11cffPCBAgICdPr0aY0caf2NRbdu3TRp0iR17NhRUVFRCggI0M8//6zk4smq06COJKnifRVVq14tTX9zujo900nF3BjYAQAAAMgNe75maPtEW82fMV+v9HxFr455VaX9Suvw/sMq7V+aa4ZsEEoBhdD+Xvvzu4RbWrBggVq2bJnlj4t04w/MlClTVLJkSX322Wd68803NXnyZHl5ealZs2aW5aKjo/X666/r5Zdf1t9//63g4GC9/vrrkqSSJUvq448/1rBhwzR//ny1aNFCkZGR6tu3723rcnBw0IoVK/TKK6+oRo0aqly5smbNmqXmzZtblnFxcdHGjRv12muvqW3btkpLS1O1atX02sTXrLb1RPcntO/Hfer0TCcBAAAABdGyFt/ndwm3ZM/XDM4uzvrgsw/0zrh39HK3l5Wenq4K91XQmLfHWG2La4YbTOZ/30hpBxISEuTt7a34+Hh5eXnl2X7s7kkbk9vldwm5VnNJzfwuIVcKUtiTnJysEydOKDQ09I73LSNvZT59L9PcaXO14csNWrV1VT5VdGfVS1W/5byCeG7Z2+9jyf5+J9vb72Mpb38nG9XWyC+F/fjuWmT2g9IWVDVDg++8UAFTkNpSOWVvf4NOFnsmv0vINXs7l+3tPE5OTtb3Px1UmcByMjm55Hc5OVKrnE9+l5Br/74muJP8vma43fVATt3uuiGnbQ3GlAIAG7maeFVHDx3V8gXL1f357vldDgAAAIAChmsGa4RSAGAjE0dO1FMtn1KDxg2KfDdcAAAAAFlxzWCNMaUAwEYmvj9RE9+fmN9lAAAAACiguGawRk8pAAAAAAAAGI5QCrBzGRkZ+V0CChk7fP4FAAAAbsss0caDjdniWpTb9wA75eLiIgcHB507d06lS5eWi4uLTCZTfpdVJGWk2l8wmJycnO10s9msixcvymQyydnZ2eCqAAAAYGsuLi7652qaPC7/JTevEjI5FvwY4FZt1YLM3q4J7uUzNpvNun79ui5evCgHBwe5uNz9Ux0L/tkIIFsODg4KDQ1VbGyszp07l9/lFGkXEi/kdwm55nT51r/+TSaTypUrJ0dHRwMrAlDYhIxcm98l5NrJYndeBgDsjYODgyZvu6RuNVNV0++aHB0K/g1TLtfc8ruEXLO3a4LbXQ/kVPHixRUcHCyHezinCKUAO+bi4qLg4GClpaUpPT09v8spsl5d9Wp+l5BrX3b68pbznJ2dCaQAAAAKkUvJGZrzY7w8XRLk7uIghwJ+g0XMa83zu4Rcs7drgttdD+SEo6OjnJyc7vluHUIpwM5l3mbFrVb5J/Z6bH6XkGvFitEdAAAAoCgxS0q4blbC9YL/ZbY9tlXt7ZqgoHzGBb/fHgAAAAAAAAodQikAAAAAAAAYjlAKAAAAAAAAhiOUAgAAAAAAgOEIpQAAAAAAAGA4QikAAAAAAAAYjlAKAAAAAAAAhiOUAgAAAAAAgOEIpQAAAAAAAGA4QikAAAAAAAAYjlAKAAAAAAAAhiOUAgAAAAAAgOEIpQAAAAAAAGA4QikAAAAAAAAYjlAKAAAAAAAAhiOUAgAAAAAAgOEIpQAAAAAAAGA4QikAAAAAAAAYjlAKAAAAAAAAhiOUAgAAAAAAgOEIpQAAAAAAAGA4QikAAAAAAAAYjlAKAAAAAAAAhrN5KJWenq6xY8cqNDRUbm5uqlixot58802ZzWbLMmazWePGjVNAQIDc3NzUsmVLHT161NalAAAA2BXaUQAAoCixeSj19ttvKzo6Wu+//74OHTqkt99+W1OmTNF7771nWWbKlCmaNWuW5s6dq127dsnd3V2tWrVScnKyrcsBAACwG7SjAABAUeJk6w3+8MMP6tChg9q1aydJCgkJ0SeffKLdu3dLuvHt3owZMzRmzBh16NBBkrR06VL5+flp9erV6tq1a5ZtpqSkKCUlxfI+ISHB1mUDAADku7xoR0m0pQAAQMFk855SjRs3VkxMjH7//XdJ0i+//KJt27apTZs2kqQTJ04oLi5OLVu2tKzj7e2tRo0aaceOHdluMyoqSt7e3pZXUFCQrcsGAADId3nRjpJoSwEAgILJ5j2lRo4cqYSEBFWpUkWOjo5KT0/XxIkT1b17d0lSXFycJMnPz89qPT8/P8u8fxs1apSGDBlieZ+QkEBjCgAAFDp50Y6SaEsBAICCyeah1H//+18tW7ZMy5cvV/Xq1bVv3z4NGjRIgYGB6tWr111t09XVVa6urjauFAAAoGDJi3aURFsKAAAUTDYPpYYNG6aRI0daxjSoWbOmTp06paioKPXq1Uv+/v6SpPPnzysgIMCy3vnz51WnTh1blwMAAGA3aEcBAICixOZjSl29elUODtabdXR0VEZGhiQpNDRU/v7+iomJscxPSEjQrl27FB4ebutyAAAA7AbtKAAAUJTYvKdU+/btNXHiRAUHB6t69er6+eefNX36dPXu3VuSZDKZNGjQIL311luqVKmSQkNDNXbsWAUGBqpjx462LgcAAMBu0I4CAABFic1Dqffee09jx47Vyy+/rAsXLigwMFAvvviixo0bZ1lm+PDhSkpKUt++fXX58mU1bdpU69evV7FixWxdDgAAgN2gHQUAAIoSm4dSnp6emjFjhmbMmHHLZUwmkyZMmKAJEybYevcAAAB2i3YUAAAoSmw+phQAAAAAAABwJ4RSAAAAAAAAMByhFAAAAAAAAAxHKAUAAAAAAADDEUoBAAAAAADAcIRSAAAAAAAAMJxTfhcAAEChFOmd3xXkTmhwflcAAACAIoaeUgAAAAAAADAcoRQAAAAAAAAMRygFAAAAAAAAwxFKAQAAAAAAwHAMdF6Y2NuguhID6wIAAAAAUETRUwoAAAAAAACGI5QCAAAAAACA4QilAAAAAAAAYDhCKQAAAAAAABiOUAoAAAAAAACGI5QCAAAAAACA4QilAAAAAAAAYDhCKQAAAAAAABiOUAoAAAAAAACGI5QCAAAAAACA4QilAAAAAAAAYDhCKQAAAAAAABiOUAoAAAAAAACGI5QCAAAAAACA4QilAAAAAAAAYDhCKQAAAAAAABiOUAoAAAAAAACGI5QCAAAAAACA4QilAAAAAAAAYDhCKQAAAAAAABiOUAoAAAAAAACGI5QCAAAAAACA4QilAAAAAAAAYDhCKQAAAAAAABjOKb8LAAAAAAAAsIj0zu8Kci80OL8rsEv0lAIAAAAAAIDhCKUAAAAAAABgOEIpAAAAAAAAGI5QCgAAAAAAAIYjlAIAAAAAAIDhCKUAAAAAAABgOEIpAAAAAAAAGI5QCgAAAAAAAIYjlAIAAAAAAIDhCKUAAAAAAABgOEIpAAAAAAAAGI5QCgAAAAAAAIYjlAIAAAAAAIDhCKUAAAAAAABgOEIpAAAAAAAAGI5QCgAAAAAAAIYjlAIAAAAAAIDhCKUAAAAAAABgOEIpAAAAAAAAGI5QCgAAAAAAAIYjlAIAAAAAAIDhCKUAAAAAAABgOEIpAAAAAAAAGI5QCgAAAAAAAIYjlAIAAAAAAIDhCKUAAAAAAABgOEIpAAAAAAAAGI5QCgAAAAAAAIYjlAIAAAAAAIDhCKUAAAAAAABgOEIpAAAAAAAAGI5QCgAAAAAAAIYjlAIAAAAAAIDhCKUAAAAAAABguDwJpf7880/16NFDvr6+cnNzU82aNbVnzx7LfLPZrHHjxikgIEBubm5q2bKljh49mhelAAAA2BXaUQAAoKiweSj1zz//qEmTJnJ2dtbXX3+tgwcPatq0aSpRooRlmSlTpmjWrFmaO3eudu3aJXd3d7Vq1UrJycm2LgcAAMBu0I4CAABFiZOtN/j2228rKChIixYtskwLDQ21/Gw2mzVjxgyNGTNGHTp0kCQtXbpUfn5+Wr16tbp27ZplmykpKUpJSbG8T0hIsHXZAAAA+S4v2lESbSkAAFAw2byn1Jdffqn69eurS5cuKlOmjOrWrav58+db5p84cUJxcXFq2bKlZZq3t7caNWqkHTt2ZLvNqKgoeXt7W15BQUG2LhsAACDf5UU7SqItBQAACiabh1J//PGHoqOjValSJW3YsEH9+vXTK6+8oiVLlkiS4uLiJEl+fn5W6/n5+Vnm/duoUaMUHx9veZ05c8bWZQMAAOS7vGhHSbSlAABAwWTz2/cyMjJUv359TZo0SZJUt25d/fbbb5o7d6569ep1V9t0dXWVq6urLcsEAAAocPKiHSXRlgIAAAWTzXtKBQQEqFq1albTqlatqtOnT0uS/P39JUnnz5+3Wub8+fOWeQAAAEUR7SgAAFCU2DyUatKkiY4cOWI17ffff1f58uUl3Ris09/fXzExMZb5CQkJ2rVrl8LDw21dDgAAgN2gHQUAAIoSm9++N3jwYDVu3FiTJk3SU089pd27d+uDDz7QBx98IEkymUwaNGiQ3nrrLVWqVEmhoaEaO3asAgMD1bFjR1uXAwAAYDdoRwEAgKLE5qFUgwYNtGrVKo0aNUoTJkxQaGioZsyYoe7du1uWGT58uJKSktS3b19dvnxZTZs21fr161WsWDFblwMAAGA3aEcBAICixOahlCQ99thjeuyxx24532QyacKECZowYUJe7B7ATUJGrs3vEnLl5OR2+V0CAOQr2lEAAKCosPmYUgAAAAAAAMCdEEoBAAAAAADAcIRSAAAAAAAAMByhFAAAAAAAAAxHKAUAAAAAAADDEUoBAAAAAADAcIRSAAAAAAAAMByhFAAAAAAAAAxHKAUAAAAAAADDEUoBAAAAAADAcIRSAAAAAAAAMByhFAAAAAAAAAxHKAUAAAAAAADDEUoBAAAAAADAcIRSAAAAAAAAMByhFAAAAAAAAAxHKAUAAAAAAADDEUoBAAAAAADAcIRSAAAAAAAAMByhFAAAAAAAAAxHKAUAAAAAAADDEUoBAAAAAADAcIRSAAAAAAAAMByhFAAAAAAAAAxHKAUAAAAAAADDEUoBAAAAAADAcIRSAAAAAAAAMByhFAAAAAAAAAxHKAUAAAAAAADDEUoBAAAAAADAcIRSAAAAAAAAMByhFAAAAAAAAAxHKAUAAAAAAADDEUoBAAAAAADAcIRSAAAAAAAAMByhFAAAAAAAAAxHKAUAAAAAAADDEUoBAAAAAADAcIRSAAAAAAAAMByhFAAAAAAAAAxHKAUAAAAAAADDEUoBAAAAAADAcIRSAAAAAAAAMByhFAAAAAAAAAxHKAUAAAAAAADDEUoBAAAAAADAcIRSAAAAAAAAMByhFAAAAAAAAAxHKAUAAAAAAADDEUoBAAAAAADAcIRSAAAAAAAAMByhFAAAAAAAAAxHKAUAAAAAAADDEUoBAAAAAADAcE75XQAAWIn0zu8Kci80OL8rAAAAAAC7Q08pAAAAAAAAGI5QCgAAAAAAAIYjlAIAAAAAAIDhCKUAAAAAAABgOEIpAAAAAAAAGI5QCgAAAAAAAIYjlAIAAAAAAIDhCKUAAAAAAABgOEIpAAAAAAAAGI5QCgAAAAAAAIYjlAIAAAAAAIDhCKUAAAAAAABgOEIpAAAAAAAAGI5QCgAAAAAAAIYjlAIAAAAAAIDhCKUAAAAAAABguDwPpSZPniyTyaRBgwZZpiUnJ6t///7y9fWVh4eHOnfurPPnz+d1KQAAAHaFdhQAACjM8jSU+vHHHzVv3jzVqlXLavrgwYP11Vdf6bPPPtPWrVt17tw5PfHEE3lZCgAAgF2hHQUAAAq7PAulEhMT1b17d82fP18lSpSwTI+Pj9eCBQs0ffp0/ec//1G9evW0aNEi/fDDD9q5c2e220pJSVFCQoLVCwAAoLCyZTtKoi0FAAAKpjwLpfr376927dqpZcuWVtP37t2r1NRUq+lVqlRRcHCwduzYke22oqKi5O3tbXkFBQXlVdkAAAD5zpbtKIm2FAAAKJjyJJRasWKFfvrpJ0VFRWWZFxcXJxcXF/n4+FhN9/PzU1xcXLbbGzVqlOLj4y2vM2fO5EXZAAAA+c7W7SiJthQAACiYnGy9wTNnzujVV1/Vpk2bVKxYMZts09XVVa6urjbZFgAAQEGVF+0oibYUAAAomGzeU2rv3r26cOGC7r//fjk5OcnJyUlbt27VrFmz5OTkJD8/P12/fl2XL1+2Wu/8+fPy9/e3dTkAAAB2g3YUAAAoSmzeU6pFixbav3+/1bTnnntOVapU0YgRIxQUFCRnZ2fFxMSoc+fOkqQjR47o9OnTCg8Pt3U5AAAAdoN2FAAAKEpsHkp5enqqRo0aVtPc3d3l6+trmd6nTx8NGTJEJUuWlJeXlwYOHKjw8HA98MADti4HAADAbtCOAgAARYnNQ6mcePfdd+Xg4KDOnTsrJSVFrVq10pw5c/KjFAAAALtCOwoAABQWhoRSW7ZssXpfrFgxzZ49W7NnzzZi9wAAAHaLdhQAACisbD7QOQAAAAAAAHAnhFIAAAAAAAAwHKEUAAAAAAAADEcoBQAAAAAAAMMRSgEAAAAAAMBwhFIAAAAAAAAwHKEUAAAAAAAADEcoBQAAAAAAAMMRSgEAAAAAAMBwhFIAAAAAAAAwHKEUAAAAAAAADEcoBQAAAAAAAMMRSgEAAAAAAMBwhFIAAAAAAAAwHKEUAAAAAAAADEcoBQAAAAAAAMMRSgEAAAAAAMBwhFIAAAAAAAAwHKEUAAAAAAAADEcoBQAAAAAAAMMRSgEAAAAAAMBwhFIAAAAAAAAwHKEUAAAAAAAADEcoBQAAAAAAAMMRSgEAAAAAAMBwhFIAAAAAAAAwHKEUAAAAAAAADEcoBQAAAAAAAMMRSgEAAAAAAMBwhFIAAAAAAAAwHKEUAAAAAAAADEcoBQAAAAAAAMMRSgEAAAAAAMBwhFIAAAAAAAAwHKEUAAAAAAAADEcoBQAAAAAAAMMRSgEAAAAAAMBwhFIAAAAAAAAwHKEUAAAAAAAADEcoBQAAAAAAAMMRSgEAAAAAAMBwhFIAAAAAAAAwHKEUAAAAAAAADEcoBQAAAAAAAMMRSgEAAAAAAMBwhFIAAAAAAAAwHKEUAAAAAAAADEcoBQAAAAAAAMMRSgEAAAAAAMBwhFIAAAAAAAAwHKEUAAAAAAAADEcoBQAAAAAAAMMRSgEAAAAAAMBwhFIAAAAAAAAwHKEUAAAAAAAADEcoBQAAAAAAAMMRSgEAAAAAAMBwhFIAAAAAAAAwHKEUAAAAAAAADEcoBQAAAAAAAMMRSgEAAAAAAMBwhFIAAAAAAAAwHKEUAAAAAAAADEcoBQAAAAAAAMMRSgEAAAAAAMBwhFIAAAAAAAAwHKEUAAAAAAAADEcoBQAAAAAAAMMRSgEAAAAAAMBwhFIAAAAAAAAwHKEUAAAAAAAADGfzUCoqKkoNGjSQp6enypQpo44dO+rIkSNWyyQnJ6t///7y9fWVh4eHOnfurPPnz9u6FAAAALtDWwoAABQVNg+ltm7dqv79+2vnzp3atGmTUlNT9eijjyopKcmyzODBg/XVV1/ps88+09atW3Xu3Dk98cQTti4FAADA7tCWAgAARYWTrTe4fv16q/eLFy9WmTJltHfvXjVr1kzx8fFasGCBli9frv/85z+SpEWLFqlq1arauXOnHnjgAVuXBAAAYDdoSwEAgKIiz8eUio+PlySVLFlSkrR3716lpqaqZcuWlmWqVKmi4OBg7dixI9ttpKSkKCEhweoFAABQFNCWAgAAhVWehlIZGRkaNGiQmjRpoho1akiS4uLi5OLiIh8fH6tl/fz8FBcXl+12oqKi5O3tbXkFBQXlZdkAAAAFAm0pAABQmOVpKNW/f3/99ttvWrFixT1tZ9SoUYqPj7e8zpw5Y6MKAQAACi7aUgAAoDCz+ZhSmQYMGKA1a9bou+++U7ly5SzT/f39df36dV2+fNnqG77z58/L398/2225urrK1dU1r0oFAAAocGhLAQCAws7mPaXMZrMGDBigVatW6ZtvvlFoaKjV/Hr16snZ2VkxMTGWaUeOHNHp06cVHh5u63IAAADsCm0pAABQVNi8p1T//v21fPly/e9//5Onp6dlbANvb2+5ubnJ29tbffr00ZAhQ1SyZEl5eXlp4MCBCg8P52kxAACgyKMtBQAAigqbh1LR0dGSpObNm1tNX7RokSIiIiRJ7777rhwcHNS5c2elpKSoVatWmjNnjq1LAQAAsDu0pQAAQFFh81DKbDbfcZlixYpp9uzZmj17tq13DwAAYNdoSwEAgKIiT5++BwAAAAAAAGSHUAoAAAAAAACGI5QCAAAAAACA4QilAAAAAAAAYDhCKQAAAAAAABiOUAoAAAAAAACGI5QCAAAAAACA4QilAAAAAAAAYDhCKQAAAAAAABiOUAoAAAAAAACGI5QCAAAAAACA4QilAAAAAAAAYDhCKQAAAAAAABiOUAoAAAAAAACGI5QCAAAAAACA4QilAAAAAAAAYDhCKQAAAAAAABiOUAoAAAAAAACGI5QCAAAAAACA4QilAAAAAAAAYDhCKQAAAAAAABiOUAoAAAAAAACGI5QCAAAAAACA4QilAAAAAAAAYDhCKQAAAAAAABiOUAoAAAAAAACGI5QCAAAAAACA4QilAAAAAAAAYDhCKQAAAAAAABiOUAoAAAAAAACGI5QCAAAAAACA4QilAAAAAAAAYDhCKQAAAAAAABiOUAoAAAAAAACGI5QCAAAAAACA4QilAAAAAAAAYDhCKQAAAAAAABiOUAoAAAAAAACGI5QCAAAAAACA4QilAAAAAAAAYDhCKQAAAAAAABiOUAoAAAAAAACGI5QCAAAAAACA4QilAAAAAAAAYDhCKQAAAAAAABiOUAoAAAAAAACGI5QCAAAAAACA4QilAAAAAAAAYDhCKQAAAAAAABiOUAoAAAAAAACGI5QCAAAAAACA4QilAAAAAAAAYDhCKQAAAAAAABiOUAoAAAAAAACGI5QCAAAAAACA4QilAAAAAAAAYDhCKQAAAAAAABiOUAoAAAAAAACGI5QCAAAAAACA4QilAAAAAAAAYDhCKQAAAAAAABiOUAoAAAAAAACGI5QCAAAAAACA4QilAAAAAAAAYDhCKQAAAAAAABiOUAoAAAAAAACGI5QCAAAAAACA4QilAAAAAAAAYDhCKQAAAAAAABiOUAoAAAAAAACGI5QCAAAAAACA4fItlJo9e7ZCQkJUrFgxNWrUSLt3786vUgAAAOwObSkAAGDv8iWU+vTTTzVkyBCNHz9eP/30k2rXrq1WrVrpwoUL+VEOAACAXaEtBQAACoN8CaWmT5+uF154Qc8995yqVaumuXPnqnjx4lq4cGF+lAMAAGBXaEsBAIDCwMnoHV6/fl179+7VqFGjLNMcHBzUsmVL7dixI9t1UlJSlJKSYnkfHx8vSUpISMjTWjNSrubp9m0twWTO7xJyLf1aen6XkCt5fc7lBc7jvGdv57Fkf+eyvZ3Hkv2dy5zH2W/bbC54/x/tpS3Fv9u8x79bY9jbuWxv57Fkf+cy53He4zzOe3l9Hue0LWV4KPXXX38pPT1dfn5+VtP9/Px0+PDhbNeJiorSG2+8kWV6UFBQntRor7zzu4C7cii/C8gV7372+SnbE/v8hO3rPJY4l41gf58w53F2rly5Im/vgvV/k7ZU3ilY/6dzgn+3yMo+P2H7Opc5j/OefX7CnMfZuVNbyvBQ6m6MGjVKQ4YMsbzPyMjQpUuX5OvrK5PJlI+V5V5CQoKCgoJ05swZeXl55Xc5wF3hPEZhwHmM2zGbzbpy5YoCAwPzuxSboC0FFCycxygMOI9xOzltSxkeSpUqVUqOjo46f/681fTz58/L398/23VcXV3l6upqNc3HxyevSjSEl5cX/3Bh9ziPURhwHuNWCloPqUy0pW7g3y4KA85jFAacx7iVnLSlDB/o3MXFRfXq1VNMTIxlWkZGhmJiYhQeHm50OQAAAHaFthQAACgs8uX2vSFDhqhXr16qX7++GjZsqBkzZigpKUnPPfdcfpQDAABgV2hLAQCAwiBfQqmnn35aFy9e1Lhx4xQXF6c6depo/fr1WQbsLIxcXV01fvz4LF3oAXvCeYzCgPMY9oy2FP92Yd84j1EYcB7DFkzmgvisYwAAAAAAABRqho8pBQAAAAAAABBKAQAAAAAAwHCEUgAAAAAAADAcoRQAAAAAAAAMRyiVQxERETKZTDKZTHJ2dlZoaKiGDx+u5ORkq+XWrFmjhx56SJ6enipevLgaNGigxYsXWy1z8uRJmUwmOTo66s8//7SaFxsbKycnJ5lMJp08edJq+exeO3fuzMvDRiGTeR6/9NJLWeb1799fJpNJERERVsv++9W6dWvLOiEhIdmeh4MGDVLz5s0t7yMjIy3rOzk5qVSpUmrWrJlmzJihlJSUPDlWICIiQh07dszyc6bPP/9cxYoV07Rp0yzL3OmcB3B3aEehMKAdhaKGthSMQCiVC61bt1ZsbKz++OMPvfvuu5o3b57Gjx9vmf/ee++pQ4cOatKkiXbt2qVff/1VXbt21UsvvaShQ4dm2V7ZsmW1dOlSq2lLlixR2bJls93/5s2bFRsba/WqV6+ebQ8ShV5QUJBWrFiha9euWaYlJydr+fLlCg4Otlo285y/+fXJJ59YLVOsWDGNGDHijvutXr26YmNjdfr0aX377bfq0qWLoqKi1LhxY125csU2Bwfk0Icffqju3bsrOjpar732mmV6Ts55AHeHdhQKA9pRwA20pWArhFK54OrqKn9/fwUFBaljx45q2bKlNm3aJEk6c+aMXnvtNQ0aNEiTJk1StWrVFBYWptdee03vvPOOpk2bpl27dlltr1evXlq0aJHVtEWLFqlXr17Z7t/X11f+/v5WL2dn57w5WBRa999/v4KCgrRy5UrLtJUrVyo4OFh169a1WjbznL/5VaJECatl+vbtq507d2rdunW33a+Tk5P8/f0VGBiomjVrauDAgdq6dat+++03vf3227Y7QOAOpkyZooEDB2rFihV67rnnrObl5JwHcHdoR6EwoB0F0JaCbRFK3aXffvtNP/zwg1xcXCTd6LqYmpqa7Td5L774ojw8PLIkxI8//rj++ecfbdu2TZK0bds2/fPPP2rfvn3eHwCKtN69e1s15BcuXJjlD0pOhYaG6qWXXtKoUaOUkZGRq3WrVKmiNm3aWDXsgLw0YsQIvfnmm1qzZo06deqU3+UARRbtKNgz2lEoymhLwdYIpXJhzZo18vDwULFixVSzZk1duHBBw4YNkyT9/vvv8vb2VkBAQJb1XFxcVKFCBf3+++9W052dndWjRw8tXLhQ0o0/aD169Ljlt3aNGzeWh4eH1Qu4Gz169NC2bdt06tQpnTp1Stu3b1ePHj2yLJd5zt/8mjRpUpblxowZoxMnTmjZsmW5rqVKlSqWcT+AvPT1119rypQp+t///qcWLVpku0xOz3kAuUc7CoUF7SgUVbSlkBec8rsAe/Lwww8rOjpaSUlJevfdd+Xk5KTOnTvf0zZ79+6txo0ba9KkSfrss8+0Y8cOpaWlZbvsp59+qqpVq97T/gBJKl26tNq1a6fFixfLbDarXbt2KlWqVJblMs/5m5UsWTLb7Q0dOlTjxo3T008/natazGazTCZT7g4AuAu1atXSX3/9pfHjx6thw4bZXpDm9JwHkHu0o1BY0I5CUUVbCnmBUCoX3N3dFRYWJunGt3G1a9fWggUL1KdPH913332Kj4/XuXPnFBgYaLXe9evXdfz4cT388MNZtlmzZk1VqVJF3bp1U9WqVVWjRg3t27cv2/0HBQVZ9g/cq969e2vAgAGSpNmzZ2e7zM3n/J0MGTJEc+bM0Zw5c3JVx6FDhxQaGpqrdYC7UbZsWX3++ed6+OGH1bp1a3399dfy9PS0WiY35zyA3KEdhcKEdhSKItpSyAvcvneXHBwc9Prrr2vMmDG6du2aOnfuLGdnZ8vjMG82d+5cJSUlqVu3btluq3fv3tqyZYt69+6d12UDFq1bt9b169eVmpqqVq1a3fP2PDw8NHbsWE2cODHHT4E5fPiw1q9ff8/flAM5Vb58eW3dulVxcXFq3bo1TywC8gntKNg72lEoqmhLwdYIpe5Bly5d5OjoqNmzZys4OFhTpkzRjBkzNHr0aB0+fFjHjx/X9OnTNXz4cL322mtq1KhRttt54YUXdPHiRT3//PO33d/ff/+tuLg4q1dycnJeHBqKAEdHRx06dEgHDx6Uo6NjtsukpKRkOef++uuvW26zb9++8vb21vLly7PMS0tLU1xcnM6dO6f9+/frvffe00MPPaQ6depYxhQBjBAUFKQtW7bowoULatWqlRISEizzcnvOA7h7tKNgz2hHoSijLQVbIpS6B05OThowYICmTJmipKQkDRo0SKtWrdL333+v+vXrq0aNGlq+fLmio6M1derU226nVKlScnK6/d2ULVu2VEBAgNVr9erVNj4qFCVeXl7y8vK65fz169dnOeeaNm16y+WdnZ315ptvZtvIP3DggAICAhQcHKzmzZvrv//9r0aNGqXvv/+ewWZhuHLlymnLli3666+/rBpTuT3nAdw92lGwd7SjUJTRloKtmMxmszm/iwAAAAAAAEDRQk8pAAAAAAAAGI5QCgAAAAAAAIYjlAIAAAAAAIDhCKUAAAAAAABgOEIpAAAAAAAAGI5QCgAAAAAAAIYjlAIAAAAAAIDhCKUAAAAAAABgOEIpAAAAAAAAGI5QCgAAAAAAAIYjlAIAAAAAAIDhCKUAAAAAAABgOEIpAAAAAAAAGI5QCgAAAAAAAIYjlAIAAAAAAIDhCKUAAAAAAABgOEIpAAAAAAAAGI5QCgAAAAAKgMWLF8tkMmnPnj13XLZ58+Zq3ry55f3JkydlMpm0ePFiy7TIyEiZTKY8qDSro0eP6tFHH5W3t7dMJpNWr15tyH4B2DdCKQAFhslkytFry5Ytt11+8uTJd9zXgQMH1KVLF1WoUEHFixdXqVKl1KxZM3311VdZlm3evLnV9l1cXBQaGqq+ffvqzJkztv4YAACADWUGPZmvYsWK6b777tOAAQN0/vx5m+5r0qRJBT6Myasae/Xqpf3792vixIn66KOPVL9+fZvvI1NmAJf5cnBwUMmSJdWmTRvt2LHjlutt375dnTp1kp+fn1xdXRUSEqIXX3xRp0+fzrJsZqDn4OCQbXsvISFBbm5uMplMGjBgwC1ru5t2KlCUOOV3AQCQ6aOPPrJ6v3TpUm3atCnL9KpVq1p+fuSRR9SzZ0+r+XXr1r3jvk6dOqUrV66oV69eCgwM1NWrV/XFF1/o8ccf17x589S3b1+r5cuVK6eoqChJ0vXr13Xw4EHNnTtXGzZs0KFDh1S8ePFcHSsAADDWhAkTFBoaquTkZG3btk3R0dFat26dfvvtN5v9HZ80aZKefPJJdezY0Sbbu52NGzfecZkxY8Zo5MiRVtPyosZr165px44dGj16tFVAk9e6deumtm3bKj09Xb///rvmzJmjhx9+WD/++KNq1qxptex7772nV199VRUqVNDAgQMVEBCgQ4cO6cMPP9Snn36qdevWqXHjxln24erqqk8++UTDhw+3mr5y5coc1fZvOWmnAkUJoRSAAqNHjx5W73fu3KlNmzZlmX6z++6777bzb6Vt27ZZGgoDBgxQvXr1NH369CyhlLe3d5b9hIaGasCAAdq+fbseeeSRXNcAAACM06ZNG0vvneeff16+vr6aPn26/ve//6lbt253vV2z2azk5GS5ubnZqtQccXFxueMyTk5OcnLK+0u+ixcvSpJ8fHxsts2kpCS5u7vfdpn777/fqn324IMPqk2bNoqOjtacOXMs07dv365BgwapadOmWr9+vVUI2a9fPzVp0kRPPvmkDhw4oBIlSljto23bttmGUsuXL1e7du30xRdf5Kg2ANnj9j0Adu/atWtKTk6+5+04OjoqKChIly9fztHy/v7+kmRIYw8AANjWf/7zH0nSiRMnJElpaWl68803VbFiRcutXa+//rpSUlKs1gsJCdFjjz2mDRs2qH79+nJzc9O8efNkMpmUlJSkJUuWWG7VioiIkHSjh/bLL7+sypUry83NTb6+vurSpYtOnjyZbW1Xr17Viy++KF9fX3l5ealnz576559/rJb595hS2fn3mFK3qvHbb7+VyWTSqlWrsmxj+fLlMplMt7wtLjIyUuXLl5ckDRs2TCaTSSEhIZb5P//8s9q0aSMvLy95eHioRYsW2rlzp9U2Mm+x3Lp1q15++WWVKVNG5cqVu+2xZefBBx+UJB0/ftxq+ptvvimTyaQlS5Zk6RVXsWJFTZkyRbGxsZo3b16WbT7zzDPat2+fDh8+bJkWFxenb775Rs8880yuawRgjSspAHZt8eLFmjNnjsxms6pWraoxY8bkqoGQlJSka9euKT4+Xl9++aW+/vprPf3001mWS09P119//SVJSk1N1aFDhzR+/HiFhYWpSZMmNjseAABgjMzgwtfXV9KN3lNLlizRk08+qddee027du1SVFSUDh06lCWsOXLkiLp166YXX3xRL7zwgipXrqyPPvpIzz//vBo2bGjpcV2xYkVJ0o8//qgffvhBXbt2Vbly5XTy5ElFR0erefPmOnjwYJagZMCAAfLx8VFkZKSOHDmi6OhonTp1Slu2bLmngctvVeMDDzygoKAgLVu2TJ06dbJaZ9myZapYsaLCw8Oz3eYTTzwhHx8fDR482HLLmoeHh6QbY3g++OCD8vLy0vDhw+Xs7Kx58+apefPm2rp1qxo1amS1rZdfflmlS5fWuHHjlJSUlOvjywz5bu7tdPXqVcXExOjBBx9UaGhotus9/fTT6tu3r9asWZPldsdmzZqpXLlyWr58uSZMmCBJ+vTTT+Xh4aF27drdsparV69a2o438/Hx4QtN4Cb8awBgtxo3bqynnnpKoaGhOnfunGbPnq3u3bsrPj5e/fr1y9E2XnvtNcu3Yg4ODnriiSf0/vvvZ1nu8OHDKl26tNW0qlWrauPGjTnqPg8AAPJXfHy8/vrrLyUnJ2v79u2aMGGC3Nzc9Nhjj+mXX37RkiVL9Pzzz2v+/PmSZOmxM3XqVH377bd6+OGHLds6duyY1q9fr1atWlnt46WXXlKFChWy3LbVrl07Pfnkk1bT2rdvr/DwcH3xxRd69tlnrea5uLgoJiZGzs7OkqTy5ctr+PDh+uqrr/T444/f9WfQo0ePW9bYo0cPTZ8+XfHx8fL29pZ047a8jRs3avTo0bfcZq1ateTl5aXBgwdnuWVtzJgxSk1N1bZt21ShQgVJUs+ePVW5cmUNHz5cW7dutdpWyZIlFRMTI0dHxxwdT2bwk56erqNHj2rIkCGSZPVZHz16VGlpaapdu/Ytt+Pq6qrKlSvr0KFDWeaZTCZ17dpVn3zyiSWUWrZsmZ544gm5urrecpvjx4/X+PHjs0zfsWOHHnjggRwdH1AUcPseALu1fft2vfrqq3r88cf10ksvae/evapRo4Zef/11Xbt2LUfbGDRokDZt2qQlS5aoTZs2Sk9P1/Xr17MsFxISok2bNmnTpk36+uuvNWPGDMXHx6tNmzaWcRQAAEDB1bJlS5UuXVpBQUHq2rWrPDw8tGrVKpUtW1br1q2TJEuokem1116TJK1du9ZqemhoaJZA6nZuHm8qNTVVf//9t8LCwuTj46Offvopy/J9+/a1BFLSjXGPnJycLHXmhZ49eyolJUWff/65Zdqnn36qtLS0uxobKT09XRs3blTHjh0tgZQkBQQE6JlnntG2bduUkJBgtc4LL7yQ40BKuhH8lC5dWv7+/nrwwQd16NAhTZs2zSqUunLliiTJ09Pzttvy9PTMUk+mZ555RseOHdOPP/5o+e+deub37dvX0na8+VWtWrUcHx9QFNBTCkCh4eLiogEDBlgCqqZNm95xnSpVqqhKlSqSbjTGHn30UbVv3167du2y6h7v7u6uli1bWt63bt1aTZs2Vf369TV58mRNmzbN9gcEAABsZvbs2brvvvvk5OQkPz8/Va5cWQ4ON76jP3XqlBwcHBQWFma1jr+/v3x8fHTq1Cmr6be6DexWrl27pqioKC1atEh//vmnzGazZV58fHyW5StVqmT13sPDQwEBAbccg8oWqlSpogYNGmjZsmXq06ePpBs9gh544IEsn0tOXLx4UVevXlXlypWzzKtataoyMjJ05swZVa9e3TI9t59r37591aVLFyUnJ+ubb77RrFmzlJ6ebrVMZhiVGU7dypUrV24ZXNWtW1dVqlTR8uXL5ePjI39/f8uYZLdSqVIlq7YjgOwRSgEoVIKCgiRJly5duqv1n3zySb344ov6/fffs21E3axevXry9vbWd999d1f7AgAAxmnYsKHl6Xu3ktPxmnL7pL2BAwdq0aJFGjRokMLDw+Xt7W25LSwjIyNX28pLPXv21KuvvqqzZ88qJSVFO3fuzHZYg7yS28/15uDnsccek6Ojo0aOHKmHH37Y8v86LCxMTk5O+vXXX2+5nZSUFB05cuS258czzzyj6OhoeXp66umnn7YEmgDuDf+SABQqf/zxhyRlGf8ppzJv+8vuW8vspKenKzEx8a72BQAACoby5csrIyNDR48etZp+/vx5Xb582fJ0uTu5Vaj1+eefq1evXpZbyx555BE1bdr0lk/8/XcdiYmJio2NtXqq3d26XfDWtWtXOTo66pNPPtGyZcvk7Oyc7QNgcqJ06dIqXry4jhw5kmXe4cOH5eDgYPky0VZGjx4tT09PjRkzxjLN3d1dDz/8sL777rssPd4y/fe//1VKSooee+yxW277mWeeUWxsrH7//XeeugfYEKEUALuU3ThOV65c0YwZM1SqVCnVq1fvtutfuHAhy7TU1FQtXbpUbm5uObrf/9tvv1ViYuJtB84EAAAFX9u2bSVJM2bMsJo+ffp0SbrtU9Zu5u7unm3Q5OjoaHXLniS99957WW41y/TBBx8oNTXV8j46OlppaWlq06ZNjuq4mxolqVSpUmrTpo0+/vhjLVu2TK1bt1apUqXuaj+Ojo569NFH9b///c/qtsPz589r+fLlatq0qby8vO5q27fi4+OjF198URs2bNC+ffss08eMGSOz2ayIiIgs446eOHFCw4cPV0BAgF588cVbbrtixYqaMWOGoqKi1LBhQ5vWDRRl3L4HwC7Nnj1bq1evVvv27RUcHKzY2FgtXLhQp0+f1kcffXTHJ+K9+OKLSkhIULNmzVS2bFnFxcVp2bJlOnz4sKZNm2Z5lHGm+Ph4ffzxx5KktLQ0y+OZ3dzcsjw6GAAA2JfatWurV69e+uCDD3T58mU99NBD2r17t5YsWaKOHTtaPXnvdurVq6fNmzdr+vTpCgwMVGhoqBo1aqTHHntMH330kby9vVWtWjXt2LFDmzdvlq+vb7bbuX79ulq0aKGnnnpKR44c0Zw5c9S0adN7evLenWrM1LNnT8tA4W+++eY97eutt97Spk2b1LRpU7388stycnLSvHnzlJKSoilTptzTtm/l1Vdf1YwZMzR58mStWLFCktSsWTNNnTpVQ4YMUa1atRQREaGAgAAdPnxY8+fPV0ZGhtatW6cSJUrccds59dNPP1najjerWLGiwsPDc3dQQCFGKAXALjVp0kQ//PCDPvzwQ/39999yd3dXw4YNtXDhwjsOPClJTz/9tBYsWKDo6Gj9/fff8vT0VL169fT2229n2+A7e/as5XHNJpNJJUqU0EMPPaTx48erTp06tj48AABgsA8//FAVKlTQ4sWLtWrVKvn7+2vUqFEaP358jrcxffp09e3bV2PGjNG1a9fUq1cvNWrUSDNnzpSjo6OWLVum5ORkNWnSRJs3b77lE/zef/99LVu2TOPGjVNqaqq6deumWbNm5XjMq7upMVP79u1VokQJZWRk3HMIVr16dX3//fcaNWqUoqKilJGRoUaNGunjjz+22qctBQYG6plnntFHH32k48ePq2LFipKkwYMHq379+po2bZrlKcoBAQHq0qWLRo8eneNbNHPqk08+0SeffJJleq9evQilgJuYzP/uRwoAAAAAKJLS0tIUGBio9u3ba8GCBfldDoBCjjGlAAAAAACSpNWrV+vixYvq2bNnfpcCoAigpxQAAAAAFHG7du3Sr7/+qjfffFOlSpXSTz/9lN8lASgC6CkFAAAAAEVcdHS0+vXrpzJlymjp0qX5XQ6AIoKeUgAAAAAAADAcPaUAAAAAAABgOEIpAAAAAAAAGM4pvwu4GxkZGTp37pw8PT1lMpnyuxwAAFDImM1mXblyRYGBgXJwKHzf4dGWAgAAeSmnbSm7DKXOnTunoKCg/C4DAAAUcmfOnFG5cuXyuwyboy0FAACMcKe2lF2GUp6enpJuHJyXl1c+VwMAAAqbhIQEBQUFWdochQ1tKQAAkJdy2payy1Aqs5u5l5cXDSkAAJBnCuutbbSlAACAEe7Ulip8gyQAAAAUAFFRUWrQoIE8PT1VpkwZdezYUUeOHLFaJjk5Wf3795evr688PDzUuXNnnT9//rbbNZvNGjdunAICAuTm5qaWLVvq6NGjeXkoAAAAeYJQCgAAIA9s3bpV/fv3186dO7Vp0yalpqbq0UcfVVJSkmWZwYMH66uvvtJnn32mrVu36ty5c3riiSduu90pU6Zo1qxZmjt3rnbt2iV3d3e1atVKycnJeX1IAAAANmUym83m/C4itxISEuTt7a34+Hi6nAMAAJvLi7bGxYsXVaZMGW3dulXNmjVTfHy8SpcureXLl+vJJ5+UJB0+fFhVq1bVjh079MADD2TZhtlsVmBgoF577TUNHTpUkhQfHy8/Pz8tXrxYXbt2zXbfKSkpSklJsTq+oKAg2lIAACBP5LQtZZdjSuVUenq6UlNT87sMFBIuLi6F8rHgAABjxMfHS5JKliwpSdq7d69SU1PVsmVLyzJVqlRRcHDwLUOpEydOKC4uzmodb29vNWrUSDt27LhlKBUVFaU33njDlocDAIDNZWRk6Pr16/ldBnLA2dlZjo6O97ydXIdS3333nd555x3t3btXsbGxWrVqlTp27GiZbzabNX78eM2fP1+XL19WkyZNFB0drUqVKlmWuXTpkgYOHKivvvpKDg4O6ty5s2bOnCkPD497PqDMGuLi4nT58mWbbA+QJAcHB4WGhsrFxSW/SwEA2JmMjAwNGjRITZo0UY0aNSRJcXFxcnFxkY+Pj9Wyfn5+iouLy3Y7mdP9/PxyvI4kjRo1SkOGDLG8z+wpBQBAQXH9+nWdOHFCGRkZ+V0KcsjHx0f+/v739GCYXIdSSUlJql27tnr37p3tmAeZ4xwsWbJEoaGhGjt2rFq1aqWDBw+qWLFikqTu3bsrNjbWMr7Cc889p759+2r58uV3fSA3ywykypQpo+LFixfaJ+fAOBkZGTp37pxiY2MVHBzMOQUAyJX+/fvrt99+07Zt2/Jl/66urnJ1dc2XfQMAcCdms1mxsbFydHRUUFAQd6gUcGazWVevXtWFCxckSQEBAXe9rVyHUm3atFGbNm1uWdiMGTM0ZswYdejQQZK0dOlS+fn5afXq1eratasOHTqk9evX68cff1T9+vUlSe+9957atm2rqVOnKjAw8K4PRrpxy15mIOXr63tP2wJuVrp0aZ07d05paWlydnbO73IAAHZiwIABWrNmjb777juVK1fOMt3f31/Xr1/X5cuXrXpLnT9/Xv7+/tluK3P6+fPnrRqA58+fV506dfKkfgAA8lpaWpquXr2qwMBAFS9ePL/LQQ64ublJki5cuKAyZcrc9a18No0f7zTOgSTt2LFDPj4+lkBKklq2bCkHBwft2rUr2+2mpKQoISHB6nUrmWNIcSLD1jJv20tPT8/nSgAA9sBsNmvAgAFatWqVvvnmG4WGhlrNr1evnpydnRUTE2OZduTIEZ0+fVrh4eHZbjM0NFT+/v5W6yQkJGjXrl23XAcAgIIu8xqLoVLsS2buci9jeds0lMrJOAdxcXEqU6aM1XwnJyeVLFnylmMhREVFydvb2/LKyRgI3F4FW+OcAgDkRv/+/fXxxx9r+fLl8vT0VFxcnOLi4nTt2jVJN76469Onj4YMGaJvv/1We/fu1XPPPafw8HCrQc6rVKmiVatWSbrxt2jQoEF666239OWXX2r//v3q2bOnAgMDrcb4BADAHnHNZV9s8f/LLp6+x+CcAADA3kRHR0uSmjdvbjV90aJFioiIkCS9++67loe+pKSkqFWrVpozZ47V8keOHLE8uU+Shg8frqSkJPXt21eXL19W06ZNtX79esvYnQAAAPbCpj2lbh7n4GY3j43g7+9vGQwrU1pami5dunTL8RNcXV3l5eVl9ULBFRISohkzZljem0wmrV69Ok/3GRERwTfEAIACxWw2Z/vKDKQkqVixYpo9e7YuXbqkpKQkrVy5Mkt76N/rmEwmTZgwQXFxcUpOTtbmzZt13333GXRUAADAXkRGRlqNOVkQr5tt2lPq5nEOMg88c5yDfv36SZLCw8N1+fJl7d27V/Xq1ZMkffPNN8rIyFCjRo1sWU4WISPX5un2b3Zycru7Wi8uLk5RUVFau3atzp49K29vb4WFhalHjx7q1auXXY6VFRsbqxIlSthkWydPnlRoaKh+/vlnq39cM2fOlNlstsk+AAAAAAD5z8hreCn31/ERERFasmSJJMnZ2VnBwcHq2bOnXn/9dTk52S5uybwOzs6OHTusbvu/nX9fNzdv3lx16tSx6lRitFx/SomJiTp27Jjl/YkTJ7Rv3z6VLFlSwcHBlnEOKlWqpNDQUI0dO9ZqnIOqVauqdevWeuGFFzR37lylpqZqwIAB6tq16z0/ec/e/fHHH2rSpIl8fHw0adIk1axZU66urtq/f78++OADlS1bVo8//ni+1GY2m5Wenn5X/7Bu1QPOlry9vfN8HwAAAAAA3Kx169ZatGiRUlJStG7dOvXv31/Ozs4aNWpUlmWvX79+T4O5b968WdWrV7ea5uvrm+P1C+J1c65v39uzZ4/q1q2runXrSpKGDBmiunXraty4cZJujHMwcOBA9e3bVw0aNFBiYmKWcQ6WLVumKlWqqEWLFmrbtq2aNm2qDz74wEaHZL9efvllOTk5ac+ePXrqqadUtWpVVahQQR06dNDatWvVvn17SdLly5f1/PPPq3Tp0vLy8tJ//vMf/fLLL5btZHbR++ijjxQSEiJvb2917dpVV65csSyTkZGhqKgohYaGys3NTbVr19bnn39umb9lyxaZTCZ9/fXXqlevnlxdXbVt2zYdP35cHTp0kJ+fnzw8PNSgQQNt3rz5tsd18+17kZGRMplMWV6LFy+WJK1fv15NmzaVj4+PfH199dhjj+n48eOWbWWmw3Xr1pXJZLKM0/HvbogpKSl65ZVXVKZMGRUrVkxNmzbVjz/+mOX4YmJiVL9+fRUvXlyNGzfWkSNHcv4/DAAAAABQpLm6usrf31/ly5dXv3791LJlS3355ZeS/v86deLEiQoMDFTlypUlSWfOnNFTTz0lHx8flSxZUh06dNDJkyfvuC9fX1/5+/tbvZydnS3zJ0+eLD8/P3l6eqpPnz5KTk62Wv/m6+aIiAht3bpVM2fOtFyX56QGW8t1KNW8efNsx0fIDBVyMs5ByZIltXz5cl25ckXx8fFauHChPDw8bHJA9urvv//Wxo0b1b9/f7m7u2e7TObI9l26dNGFCxf09ddfa+/evbr//vvVokULXbp0ybLs8ePHtXr1aq1Zs0Zr1qzR1q1bNXnyZMv8qKgoLV26VHPnztWBAwc0ePBg9ejRQ1u3brXa58iRIzV58mQdOnRItWrVUmJiotq2bauYmBj9/PPPat26tdq3b6/Tp0/n6DiHDh2q2NhYy2vq1KkqXry46tevL0lKSkrSkCFDtGfPHsXExMjBwUGdOnVSRkaGJGn37t2SbiTEsbGxWrlyZbb7GT58uL744gstWbJEP/30k8LCwtSqVSurz0iSRo8erWnTpmnPnj1ycnJS7969c3QcAAAAAAD8m5ubm65fv255HxMToyNHjmjTpk1as2aNUlNT1apVK3l6eur777/X9u3b5eHhodatW1utl1v//e9/FRkZqUmTJmnPnj0KCAjI8vCUm82cOVPh4eF64YUXLNfn+fFAObt4+l5RcOzYMZnNZktymqlUqVKWdLN///5q3769du/erQsXLsjV1VWSNHXqVK1evVqff/65+vbtK+lGT6jFixfL09NTkvTss88qJiZGEydOVEpKiiZNmqTNmzcrPDxcklShQgVt27ZN8+bN00MPPWTZ/4QJE/TII49Y3pcsWVK1a9e2vH/zzTe1atUqffnllxowYMAdj9PDw8MSQO7cuVNjxozRkiVLVKNGDUlS586drZZfuHChSpcurYMHD6pGjRoqXbq0pP9PiLOTlJSk6OhoLV68WG3atJEkzZ8/X5s2bdKCBQs0bNgwy7ITJ060HO/IkSPVrl07JScn8wQjAAAAAECOmc1mxcTEaMOGDRo4cKBluru7uz788EPLbXsff/yxMjIy9OGHH1o6nixatEg+Pj7asmWLHn300Vvuo3HjxnJwsO5blJiYKEmaMWOG+vTpoz59+kiS3nrrLW3evDlLb6lM3t7ecnFxUfHixQ0ZcudWCKUKuN27dysjI0Pdu3dXSkqKfvnlFyUmJma5b/TatWtWt7mFhIRYAilJCggIsDz18NixY7p69apV2CTduL8187bMTJk9mDIlJiYqMjJSa9euVWxsrNLS0nTt2rUc95TKdPr0aXXs2FFDhw7VU089ZZl+9OhRjRs3Trt27dJff/1l6SF1+vRpS3B1J8ePH1dqaqqaNGlimebs7KyGDRvq0KFDVsvWqlXL8nNAQIAk6cKFCwoODs7V8QAAAAAAip41a9bIw8NDqampysjI0DPPPKPIyEjL/Jo1a1qNI/XLL7/o2LFjVtfrkpScnGx1TZ+dTz/9VFWrVs123qFDh/TSSy9ZTQsPD9e3336byyMyFqFUAREWFiaTyZRlTKMKFSpIutEFULoRCgUEBGjLli1ZtuHj42P5+eb7SqUbt/5lBjyZSeratWtVtmxZq+Uye19l+vethEOHDtWmTZs0depUhYWFyc3NTU8++WSuuhkmJSXp8ccfV3h4uCZMmGA1r3379ipfvrzmz5+vwMBAZWRkqEaNGvfUjfF2bv6cMlPqzM8JAIqSmktq5ncJuba/1/78LgG3YfQTkwqau30SMwDAvjz88MOKjo6Wi4uLAgMDszwc7N/X1ImJiapXr56WLVuWZVuZdwbdSlBQkMLCwu696AKEUKqA8PX11SOPPKL3339fAwcOvOW4Uvfff7/i4uLk5OSkkJCQu9pXtWrV5OrqqtOnT1vdqpcT27dvV0REhDp16iTpxj+o3AyGZjab1aNHD2VkZOijjz6yBEHSjXG1jhw5ovnz5+vBBx+UJG3bts1q/cyEOT09/Zb7qFixolxcXLR9+3aVL19ekpSamqoff/xRgwYNynGtAAAAAADcjru7e66Covvvv1+ffvqpypQpIy8vL5vVUbVqVe3atUs9e/a0TNu5c+dt13FxcbnttbURcj3QOfLOnDlzlJaWpvr16+vTTz/VoUOHdOTIEX388cc6fPiwHB0d1bJlS4WHh6tjx47auHGjTp48qR9++EGjR4/Wnj17crQfT09PDR06VIMHD9aSJUt0/Phx/fTTT3rvvfe0ZMmS265bqVIlrVy5Uvv27dMvv/yiZ555Jlc9iyIjI7V582bNmzdPiYmJiouLU1xcnK5du6YSJUrI19dXH3zwgY4dO6ZvvvlGQ4YMsVq/TJkycnNz0/r163X+/HnFx8dn2Ye7u7v69eunYcOGaf369Tp48KBeeOEFXb161XJ/LQAAAAAARuvevbtKlSqlDh066Pvvv9eJEye0ZcsWvfLKKzp79uxt1/37778t19CZr8wxo1599VUtXLhQixYt0u+//67x48frwIEDt91eSEiIdu3apZMnT1oNn2OkItVTqqB3o65YsaJ+/vlnTZo0SaNGjdLZs2fl6uqqatWqaejQoXr55ZdlMpm0bt06jR49Ws8995wuXrwof39/NWvWTH5+fjne15tvvqnSpUsrKipKf/zxh3x8fHT//ffr9ddfv+1606dPV+/evdW4cWOVKlVKI0aMUEJCQo73u3XrViUmJqpx48ZW0xctWqSIiAitWLFCr7zyimrUqKHKlStr1qxZat68uWU5JycnzZo1SxMmTNC4ceP04IMPZnsr4+TJk5WRkaFnn31WV65cUf369bVhwwaVKFEix7UCAAAAAPJPQb+GvxvFixfXd999pxEjRuiJJ57QlStXVLZsWbVo0eKOPadatmyZZdonn3yirl276umnn9bx48c1fPhwJScnq3PnzurXr582bNhwy+0NHTpUvXr1UrVq1XTt2jWdOHHiru/Iulsms9lsNnSPNpCQkCBvb2/Fx8dn+Z+WnJysEydOKDQ0lCeowaY4twAUZowpZe12bY3CwIjjY0ypwnchBQB5hWst+3S7/285bWtw+x4AAAAAAAAMRygFAAAAAAAAwxFKAQAAAAAAwHCEUgAAAAAAADAcoRQAAAAAAAAMRygFAAAAAAAAwxFKAQAAAAAAwHCEUgAAAAAAADAcoRQAAAAAAAAMRygFSVJkZKTq1KljeR8REaGOHTvmWz0AAAAAACBv/Puav3nz5ho0aJDhdTgZvsf8FOlt4L7ic71KRESELl++rNWrV9u+nlyaOXOmzGaz5X3z5s1Vp04dzZgxI/+KAgAAAAAUHUZew0u5vo6PiIjQkiVLJEnOzs4KDg5Wz5499frrr8vJ6d7ili1btujhhx/WP//8Ix8fnzsul53Y2Fj5+/vnaH8rV66Us7Oz5X1ISIgGDRqU50FV0QqlkGPe3gb/4wcAAAAAwM60bt1aixYtUkpKitatW6f+/fvL2dlZo0aNMrSOI0eOyMvLy2pamTJlcrx+yZIlbV1SjnD7np3YunWrGjZsKFdXVwUEBGjkyJFKS0uzzM/IyNCUKVMUFhYmV1dXBQcHa+LEiZb5I0aM0H333afixYurQoUKGjt2rFJTU2+5v5u78kVERGjr1q2aOXOmTCaTTCaTTpw4obCwME2dOtVqvX379slkMunYsWO2/QAAAAAAAChgXF1d5e/vr/Lly6tfv35q2bKlvvzyS0nSP//8o549e6pEiRIqXry42rRpo6NHj1rWPXXqlNq3b68SJUrI3d1d1atX17p163Ty5ElL76cSJUrIZDIpIiLitnWUKVNG/v7+Vi8HhxuRT3p6uoYMGSIfHx/5+vpq+PDhVndGSda37zVv3lynTp3S4MGDLRlAXiGUsgN//vmn2rZtqwYNGuiXX35RdHS0FixYoLfeesuyzKhRozR58mSNHTtWBw8e1PLly+Xn52eZ7+npqcWLF+vgwYOaOXOm5s+fr3fffTdH+585c6bCw8P1wgsvKDY2VrGxsQoODlbv3r21aNEiq2UXLVqkZs2aKSwszDYHDwAAAACAnXBzc9P169cl3ejgsWfPHn355ZfasWOHzGaz2rZta+kg0r9/f6WkpOi7777T/v379fbbb8vDw0NBQUH64osvJN3oARUbG6uZM2fedU3Tpk3T4sWLtXDhQm3btk2XLl3SqlWrbrn8ypUrVa5cOU2YMMGSAeQVbt+zA3PmzFFQUJDef/99mUwmValSRefOndOIESM0btw4JSUlaebMmXr//ffVq1cvSVLFihXVtGlTyzbGjBlj+TkkJERDhw7VihUrNHz48Dvu39vbWy4uLipevLjV/agREREaN26cdu/erYYNGyo1NVXLly/P0nsK+Stk5Nr8LiFXTk5ul98lAAAAAECumM1mxcTEaMOGDRo4cKCOHj2qL7/8Utu3b1fjxo0lScuWLVNQUJBWr16tLl266PTp0+rcubNq1qwpSapQoYJle5m305UpU+a2Y0plKleunNX78uXL68CBA5KkGTNmaNSoUXriiSckSXPnztWGDRtuua2SJUvK0dFRnp6eOR6T6m4RStmBQ4cOKTw83KrLXJMmTZSYmKizZ88qLi5OKSkpatGixS238emnn2rWrFk6fvy4EhMTlZaWluV+09wKDAxUu3bttHDhQjVs2FBfffWVUlJS1KVLl3vaLgAAAAAA9mDNmjXy8PBQamqqMjIy9MwzzygyMlIxMTFycnJSo0aNLMv6+vqqcuXKOnTokCTplVdeUb9+/bRx40a1bNlSnTt3Vq1ate6qju+//16enp6W95mDlsfHxys2NtaqDicnJ9WvXz/LLXz5gdv3CgE3N7fbzt+xY4e6d++utm3bas2aNfr55581evRoS5fCe/H8889rxYoVunbtmhYtWqSnn35axYsXv+ftAgAAAABQ0D388MPat2+fjh49qmvXrmnJkiVyd3fP0brPP/+8/vjjDz377LPav3+/6tevr/fee++u6ggNDVVYWJjlVb58+bvajtEIpexA1apVLfefZtq+fbs8PT1Vrlw5VapUSW5uboqJicl2/R9++EHly5fX6NGjVb9+fVWqVEmnTp3KVQ0uLi5KT0/PMr1t27Zyd3dXdHS01q9fr969e+fu4AAAAAAAsFPu7u4KCwtTcHCwnJz+/2a0qlWrKi0tTbt27bJM+/vvv3XkyBFVq1bNMi0oKEgvvfSSVq5cqddee03z58+XdOMaXFK21+G54e3trYCAAKs60tLStHfv3tuud6sMwNa4fa+AiY+P1759+6ym9e3bVzNmzNDAgQM1YMAAHTlyROPHj9eQIUPk4OCgYsWKacSIERo+fLhcXFzUpEkTXbx4UQcOHFCfPn1UqVIlnT59WitWrFCDBg20du3a2w5qlp2QkBDt2rVLJ0+elIeHh0qWLCkHBwc5OjoqIiJCo0aNUqVKlRQeHm7DTwMAAAAAAPtTqVIldejQQS+88ILmzZsnT09PjRw5UmXLllWHDh0kSYMGDVKbNm1033336Z9//tG3336rqlWrSroxJpTJZNKaNWvUtm1bubm5ycPD45b7u3DhgpKTk62m+fr6ytnZWa+++qomT56sSpUqqUqVKpo+fbouX7582/pDQkL03XffqWvXrnJ1dVWpUqXu7QO5haIVSkXG53cFd7RlyxbVrVvXalqfPn20bt06DRs2TLVr11bJkiXVp08fq8HLx44dKycnJ40bN07nzp1TQECAXnrpJUnS448/rsGDB2vAgAFKSUlRu3btNHbsWEVGRua4rqFDh/5fe/ce5lVZ743/PZwGDzB4QAZsFEwDFVAzRdQSgx2gmZiPO9n6gOcycGuYB3w8kFqobQUNRDMEvcpj20NbjTIK3CZ4wCgsJTUQTAYPW4agGBDm90c/ZzdxUGRmfQFfr+u6r8u11r3u9VlwO7N4f9da3wwdOjT77LNP/va3v2XevHnp3LlzfX3f+c53cuqpp27y+QMAAECSLeLf8BsyadKknHvuufniF7+YlStX5nOf+1wee+yx+vc9rV69OsOGDcvrr7+etm3bZsCAARkzZkySZNddd823vvWtXHzxxTn11FMzZMiQTJ48eb3H6tq161rrZsyYkUMOOSTnn39+Fi1alKFDh6ZZs2Y57bTTctxxx6WmZv1/vldeeWW++tWv5pOf/GRqa2ub7P1TZXWbw5utNtLSpUtTUVGRmpqatV7WvWLFisybNy9dunRJ69atS1Thx8t///d/p2/fvlm4cGE6dOhQ6nKazJY6t7a4b99r/W+lLmHjbeG/LCFJetzRo9QlbLQ5Q+c02dgbutbYGhRxflva75/G5ttkAT68LfXfWh93G/p7+7DXGh+vO6VoVLW1tXnrrbcyatSonHDCCVt1IAUAAAA0Li865yO7++67s/vuu2fJkiW57rrrSl0OAAAAsAURSvGRnXLKKVm9enVmzZqVXXfdtdTlAAAAAFsQoRQAAAAAhRNKAQAAACW3BX4P28daY/x9CaUAAACAkmnevHmSZOXKlSWuhI3x17/+NUnSsmXLjzyGb98DAAAASqZFixbZdttt89Zbb6Vly5Zp1sz9M5uzurq6/PWvf82bb76Zdu3a1YeKH4VQCgAAACiZsrKydOzYMfPmzctrr71W6nL4kNq1a5fKyspNGkMoBQAAAJRUq1atstdee3mEbwvRsmXLTbpD6n1CKQAAAKDkmjVrltatW5e6DAr0sQqletzRo7BjzRk65yPvO2PGjBx++OEZMGBAHn300UasCgAo0hNPPJHvfve7mTVrVhYtWpQHH3wwgwYNqt9eVla2zv2uu+66XHDBBevcNmrUqHzrW99qsK5r16556aWXGq1uAIAieHvYZmjixIk555xz8sQTT+SNN94oWR1umwSATbN8+fLst99+GT9+/Dq3L1q0qEG7/fbbU1ZWluOPP36D4+67774N9nvyySebonwAgCYllNrMLFu2LPfee2/OPvvsHH300Zk8eXKD7f/1X/+Vgw46KK1bt87OO++c4447rn5bbW1tLrroolRVVaW8vDx77rlnJk6cmCSZPHly2rVr12Cshx56qMEntKNGjcr++++fH/zgB+nSpUv9bZNTpkzJ4Ycfnnbt2mWnnXbKF7/4xbz66qsNxnr99dczePDg7Ljjjtluu+3ymc98Jk8//XTmz5+fZs2a5bnnnmvQf+zYsdl9992zZs2aTf0jA4DN1sCBA3P11Vc3+H39jyorKxu0hx9+OEceeWT22GOPDY7bokWLBvvtvPPOG+xfW1ubpUuXNmgAAKUmlNrM3HfffenWrVu6du2ak08+Obfffnvq6uqSJI8++miOO+64HHXUUfnNb36TqVOn5uCDD67fd8iQIbn77rtz00035cUXX8ytt96a7bfffqOO/8orr+Q///M/88ADD2T27NlJ/v4p74gRI/Lcc89l6tSpadasWY477rj6QGnZsmU54ogj8uc//zk/+clP8tvf/jYXXnhh1qxZk86dO6dfv36ZNGlSg+NMmjQpp5xyiq/6BID/3+LFi/Poo4/m9NNP/8C+L7/8cjp16pQ99tgjJ510UhYsWLDB/qNHj05FRUV9q6qqaqyyAQA+so/VO6W2BBMnTszJJ5+cJBkwYEBqamoyffr09OnTJ9/+9rdz4oknNniPxH777Zck+eMf/5j77rsvjz/+ePr165ckH/gp67qsXLkyd955Z9q3b1+/7p8fIbj99tvTvn37/OEPf0j37t1z11135a233sqzzz6bHXfcMUmy55571vc/44wz8rWvfS033HBDysvL8/zzz2fOnDl5+OGHN7o+ANha3XHHHWnTpk2+/OUvb7Bfr169Mnny5HTt2jWLFi3Kt771rXz2s5/NCy+8kDZt2qxzn5EjR2bEiBH1y0uXLhVMAQAl5zaVzcjcuXPzzDPPZPDgwUn+fmv+V77ylfpH8GbPnp2+ffuuc9/Zs2enefPmOeKIIzapht13371BIJX8/dPYwYMHZ4899kjbtm3TuXPnJKn/VHb27Nk54IAD6gOpfzZo0KA0b948Dz74YJK/P0p45JFH1o8DAPz9Q5+TTjrpA791aODAgTnhhBPSs2fP9O/fP4899liWLFmS++67b737lJeXp23btg0aAECpuVNqMzJx4sS899576dSpU/26urq6lJeXZ9y4cdlmm23Wu++GtiV//2rN9x8DfN+qVavW6rfddtutte6YY47J7rvvnttuuy2dOnXKmjVr0r179/oXoX/QsVu1apUhQ4Zk0qRJ+fKXv5y77rorN9544wb3AYCPk//+7//O3Llzc++99270vu3atcunPvWpvPLKK01QGQBA03Gn1Gbivffey5133pnrr78+s2fPrm+//e1v06lTp9x9993p2bNnpk6dus79e/TokTVr1mT69Onr3N6+ffv85S9/yfLly+vXvf/OqA155513Mnfu3Fx66aXp27dv9t5777z77rsN+vTs2TOzZ8/O//zP/6x3nDPOOCO/+MUvcvPNN+e99977wEcTAODjZOLEiTnwwAPrH8vfGMuWLcurr76ajh07NkFlAABNRyi1mXjkkUfy7rvv5vTTT0/37t0btOOPPz4TJ07MFVdckbvvvjtXXHFFXnzxxcyZMyfXXnttkqRz584ZOnRoTjvttDz00EOZN29epk2bVn8rf69evbLtttvmkksuyauvvpq77rprrW/2W5cddtghO+20U77//e/nlVdeyS9/+csG76RIksGDB6eysjKDBg3Kr3/96/zpT3/Kf/7nf2bGjBn1ffbee+8ccsghueiiizJ48OAPvLsKALYGy5Ytq/+gKUnmzZuX2bNnN3gx+dKlS3P//ffnjDPOWOcYffv2zbhx4+qXv/nNb2b69OmZP39+nnrqqRx33HFp3rx5/eP/AABbio/V43tzhs4pdQnrNXHixPTr1y8VFRVrbTv++ONz3XXXZccdd8z999+fq666Ktdcc03atm2bz33uc/X9JkyYkEsuuSRf//rX884772S33XbLJZdckiTZcccd88Mf/jAXXHBBbrvttvTt2zejRo3KWWedtcG6mjVrlnvuuSf//u//nu7du6dr16656aab0qdPn/o+rVq1ys9//vOcf/75Oeqoo/Lee+9ln332yfjx4xuMdfrpp+epp57Kaaedtgl/UgCw5Xjuuedy5JFH1i+//8HO0KFD6z8cuueee1JXV7feUOnVV1/N22+/Xb/8+uuvZ/DgwXnnnXfSvn37HH744Zk5c+Za74QEANjcldX984uGtgBLly5NRUVFampq1npR54oVKzJv3rx06dLlA18USrGuuuqq3H///fnd735X6lI+ki11bnW++NFSl7BR5rf+t1KXsPFG1ZS6AthkPe7oUeoSNlpTfti0oWuNrUER57el/f5pbPOvObrUJQBAyXzYaw2P79Hkli1blhdeeCHjxo3LOeecU+pyAAAAgM2AUIomN3z48Bx44IHp06ePR/cAAACAJB+zd0pRGpMnT/5QL1UHAAAAPj7cKQUAAABA4bbaUGoLfH87mzlzCgAAABrPVhdKtWzZMkny17/+tcSVsLVZuXJlkqR58+YlrgQAAAC2fFvdO6WaN2+edu3a5c0330ySbLvttikrKytxVWzp1qxZk7feeivbbrttWrTY6v63AQAAgMJtlf+6rqysTJL6YAoaQ7NmzbLbbrsJOQEAAKARbJWhVFlZWTp27Jhddtklq1atKnU5bCVatWqVZs22uideAQAAoCS2ylDqfc2bN/f+HwAAAIDNkNs+AAAAACicUAoAAACAwgmlAAAAACicUAoAAACAwm3VLzoHYOvQ+eJHS13CRpt/zdGlLgEAADZr7pQCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHAtSl0AwJauxx09Sl3CRpszdE6pSwAAAD7m3CkFAAAAQOEaPZRavXp1LrvssnTp0iXbbLNNPvnJT+aqq65KXV1dfZ+6urpcfvnl6dixY7bZZpv069cvL7/8cmOXAgAAAMBmqtFDqWuvvTYTJkzIuHHj8uKLL+baa6/Nddddl+9973v1fa677rrcdNNNueWWW/L0009nu+22S//+/bNixYrGLgcAAACAzVCjv1PqqaeeyrHHHpujjz46SdK5c+fcfffdeeaZZ5L8/S6psWPH5tJLL82xxx6bJLnzzjvToUOHPPTQQznxxBMbuyQAAAAANjONfqfUoYcemqlTp+aPf/xjkuS3v/1tnnzyyQwcODBJMm/evFRXV6dfv371+1RUVKRXr16ZMWPGOsesra3N0qVLGzQAAAAAtlyNfqfUxRdfnKVLl6Zbt25p3rx5Vq9enW9/+9s56aSTkiTV1dVJkg4dOjTYr0OHDvXb/tno0aPzrW99q7FLBQAAAKBEGv1Oqfvuuy8/+tGPctddd+X555/PHXfckf/4j//IHXfc8ZHHHDlyZGpqaurbwoULG7FiAAAAAIrW6HdKXXDBBbn44ovr3w3Vo0ePvPbaaxk9enSGDh2aysrKJMnixYvTsWPH+v0WL16c/ffff51jlpeXp7y8vLFLBQAAAKBEGv1Oqb/+9a9p1qzhsM2bN8+aNWuSJF26dEllZWWmTp1av33p0qV5+umn07t378YuBwAAAIDNUKPfKXXMMcfk29/+dnbbbbfsu++++c1vfpMbbrghp512WpKkrKws5513Xq6++urstdde6dKlSy677LJ06tQpgwYNauxyAAAAANgMNXoo9b3vfS+XXXZZvv71r+fNN99Mp06d8tWvfjWXX355fZ8LL7wwy5cvz1lnnZUlS5bk8MMPz5QpU9K6devGLgcAAACAzVCjh1Jt2rTJ2LFjM3bs2PX2KSsry5VXXpkrr7yysQ8PAAAAwBag0d8pBQAAAAAfRCgFAAAAQOGEUgAAAAAUTigFAAAAQOGEUgAAAAAUTigFAAAAQOGEUgAAAAAUTigFANBEnnjiiRxzzDHp1KlTysrK8tBDDzXYfsopp6SsrKxBGzBgwAeOO378+HTu3DmtW7dOr1698swzzzTRGQAANB2hFABAE1m+fHn222+/jB8/fr19BgwYkEWLFtW3u+++e4Nj3nvvvRkxYkSuuOKKPP/889lvv/3Sv3//vPnmm41dPgBAk2pR6gIAALZWAwcOzMCBAzfYp7y8PJWVlR96zBtuuCFnnnlmTj311CTJLbfckkcffTS33357Lr744nXuU1tbm9ra2vrlpUuXfujjAQA0FXdKAQCU0LRp07LLLruka9euOfvss/POO++st+/KlSsza9as9OvXr35ds2bN0q9fv8yYMWO9+40ePToVFRX1raqqqlHPAQDgoxBKAQCUyIABA3LnnXdm6tSpufbaazN9+vQMHDgwq1evXmf/t99+O6tXr06HDh0arO/QoUOqq6vXe5yRI0empqamvi1cuLBRzwMA4KPw+B4AQImceOKJ9f/do0eP9OzZM5/85Cczbdq09O3bt9GOU15envLy8kYbDwCgMbhTCgBgM7HHHntk5513ziuvvLLO7TvvvHOaN2+exYsXN1i/ePHijXovFQDA5kAoBQCwmXj99dfzzjvvpGPHjuvc3qpVqxx44IGZOnVq/bo1a9Zk6tSp6d27d1FlAgA0CqEUAEATWbZsWWbPnp3Zs2cnSebNm5fZs2dnwYIFWbZsWS644ILMnDkz8+fPz9SpU3Psscdmzz33TP/+/evH6Nu3b8aNG1e/PGLEiNx2222544478uKLL+bss8/O8uXL67+NDwBgS+GdUgAATeS5557LkUceWb88YsSIJMnQoUMzYcKE/O53v8sdd9yRJUuWpFOnTvnCF76Qq666qsH7n1599dW8/fbb9ctf+cpX8tZbb+Xyyy9PdXV19t9//0yZMmWtl58DAGzuhFIAAE2kT58+qaurW+/2n/3sZx84xvz589daN3z48AwfPnxTSgMAKDmP7wEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEANJEnnngixxxzTDp16pSysrI89NBD9dtWrVqViy66KD169Mh2222XTp06ZciQIXnjjTc2OOaoUaNSVlbWoHXr1q2JzwQAoPEJpQAAmsjy5cuz3377Zfz48Wtt++tf/5rnn38+l112WZ5//vk88MADmTt3br70pS994Lj77rtvFi1aVN+efPLJpigfAKBJtSh1AQAAW6uBAwdm4MCB69xWUVGRxx9/vMG6cePG5eCDD86CBQuy2267rXfcFi1apLKyslFrBQAomjulAAA2EzU1NSkrK0u7du022O/ll19Op06dsscee+Skk07KggULNti/trY2S5cubdAAAEpNKAUAsBlYsWJFLrroogwePDht27Zdb79evXpl8uTJmTJlSiZMmJB58+bls5/9bP7yl7+sd5/Ro0enoqKivlVVVTXFKQAAbBShFABAia1atSr/+q//mrq6ukyYMGGDfQcOHJgTTjghPXv2TP/+/fPYY49lyZIlue+++9a7z8iRI1NTU1PfFi5c2NinAACw0bxTCgCghN4PpF577bX88pe/3OBdUuvSrl27fOpTn8orr7yy3j7l5eUpLy/f1FIBABqVO6UAAErk/UDq5Zdfzi9+8YvstNNOGz3GsmXL8uqrr6Zjx45NUCEAQNMRSgEANJFly5Zl9uzZmT17dpJk3rx5mT17dhYsWJBVq1bl//yf/5PnnnsuP/rRj7J69epUV1enuro6K1eurB+jb9++GTduXP3yN7/5zUyfPj3z58/PU089leOOOy7NmzfP4MGDiz49AIBN4vE9AIAm8txzz+XII4+sXx4xYkSSZOjQoRk1alR+8pOfJEn233//Bvv96le/Sp8+fZIkr776at5+++36ba+//noGDx6cd955J+3bt8/hhx+emTNnpn379k17MgAAjUwoBQDQRPr06ZO6urr1bt/QtvfNnz+/wfI999yzqWUBAGwWPL4HAAAAQOGEUgAAAAAUrklCqT//+c85+eSTs9NOO2WbbbZJjx498txzz9Vvr6ury+WXX56OHTtmm222Sb9+/fLyyy83RSkAAAAAbIYaPZR69913c9hhh6Vly5b56U9/mj/84Q+5/vrrs8MOO9T3ue6663LTTTfllltuydNPP53tttsu/fv3z4oVKxq7HAAAAAA2Q43+ovNrr702VVVVmTRpUv26Ll261P93XV1dxo4dm0svvTTHHntskuTOO+9Mhw4d8tBDD+XEE09s7JIAAAAA2Mw0+p1SP/nJT/KZz3wmJ5xwQnbZZZcccMABue222+q3z5s3L9XV1enXr1/9uoqKivTq1SszZsxY55i1tbVZunRpgwYAAADAlqvRQ6k//elPmTBhQvbaa6/87Gc/y9lnn51///d/zx133JEkqa6uTpJ06NChwX4dOnSo3/bPRo8enYqKivpWVVXV2GUDAAAAUKBGD6XWrFmTT3/60/nOd76TAw44IGeddVbOPPPM3HLLLR95zJEjR6ampqa+LVy4sBErBgAAAKBojR5KdezYMfvss0+DdXvvvXcWLFiQJKmsrEySLF68uEGfxYsX12/7Z+Xl5Wnbtm2DBgAAAMCWq9FDqcMOOyxz585tsO6Pf/xjdt999yR/f+l5ZWVlpk6dWr996dKlefrpp9O7d+/GLgcAAACAzVCjf/veN77xjRx66KH5zne+k3/913/NM888k+9///v5/ve/nyQpKyvLeeedl6uvvjp77bVXunTpkssuuyydOnXKoEGDGrscAAAAADZDjR5KHXTQQXnwwQczcuTIXHnllenSpUvGjh2bk046qb7PhRdemOXLl+ess87KkiVLcvjhh2fKlClp3bp1Y5cDAAAAwGao0UOpJPniF7+YL37xi+vdXlZWliuvvDJXXnllUxweAAAAgM1co79TCgAAAAA+iFAKAAAAgMIJpQAAAAAonFAKAAAAgMIJpQAAAAAonFAKAAAAgMIJpQAAAAAonFAKAAAAgMIJpQAAAAAonFAKAAAAgMIJpQAAAAAonFAKAAAAgMIJpQAAAAAonFAKAAAAgMIJpQAAAAAonFAKAAAAgMIJpQAAAAAonFAKAAAAgMK1KHUBAAAAQPE6X/xoqUsoqfnXHF3qEj723CkFAAAAQOGEUgAAAAAUTigFAAAAQOGEUgAAAAAUTigFAAAAQOGEUgAAAAAUTigFAAAAQOGEUgAAAAAUTigFAAAAQOGEUgAAAAAUTigFANBEnnjiiRxzzDHp1KlTysrK8tBDDzXYXldXl8svvzwdO3bMNttsk379+uXll1/+wHHHjx+fzp07p3Xr1unVq1eeeeaZJjoDAICmI5QCAGgiy5cvz3777Zfx48evc/t1112Xm266KbfcckuefvrpbLfddunfv39WrFix3jHvvffejBgxIldccUWef/757Lfffunfv3/efPPNpjoNAIAmIZQCAGgiAwcOzNVXX53jjjturW11dXUZO3ZsLr300hx77LHp2bNn7rzzzrzxxhtr3VH1j2644YaceeaZOfXUU7PPPvvklltuybbbbpvbb7+9Cc8EAKDxCaUAAEpg3rx5qa6uTr9+/erXVVRUpFevXpkxY8Y691m5cmVmzZrVYJ9mzZqlX79+690nSWpra7N06dIGDQCg1FqUugAAgI+j6urqJEmHDh0arO/QoUP9tn/29ttvZ/Xq1evc56WXXlrvsUaPHp1vfetbm1gxAGxlRlWUuoLSGVVT6gqSuFMKAGCrN3LkyNTU1NS3hQsXlrokAAChFABAKVRWViZJFi9e3GD94sWL67f9s5133jnNmzffqH2SpLy8PG3btm3QAABKTSgFAFACXbp0SWVlZaZOnVq/bunSpXn66afTu3fvde7TqlWrHHjggQ32WbNmTaZOnbrefQAANlfeKQUA0ESWLVuWV155pX553rx5mT17dnbcccfstttuOe+883L11Vdnr732SpcuXXLZZZelU6dOGTRoUP0+ffv2zXHHHZfhw4cnSUaMGJGhQ4fmM5/5TA4++OCMHTs2y5cvz6mnnlr06QEAbBKhFABAE3nuuedy5JFH1i+PGDEiSTJ06NBMnjw5F154YZYvX56zzjorS5YsyeGHH54pU6akdevW9fu8+uqrefvtt+uXv/KVr+Stt97K5Zdfnurq6uy///6ZMmXKWi8/BwDY3AmlAACaSJ8+fVJXV7fe7WVlZbnyyitz5ZVXrrfP/Pnz11o3fPjw+junAAC2VN4pBQAAAEDhhFIAAAAAFE4oBQAAAEDhhFIAAAAAFE4oBQAAAEDhhFIAAAAAFE4oBQAAAEDhhFIAAAAAFE4oBQAAAEDhhFIAAAAAFE4oBQAAAEDhhFIAAAAAFE4oBQAAAEDhhFIAAAAAFE4oBQAAAEDhhFIAAAAAFE4oBQAAAEDhhFIAAAAAFE4oBQAAAEDhhFIAAAAAFE4oBQAAAEDhWpS6AAAAADZzoypKXUFpjaopdQWwVXKnFAAAAACFE0oBAAAAUDihFAAAAACFE0oBAAAAUDihFAAAAACFE0oBAAAAUDihFAAAAACFE0oBAAAAUDihFAAAAACFE0oBAAAAUDihFAAAAACFE0oBAAAAULgmD6WuueaalJWV5bzzzqtft2LFigwbNiw77bRTtt9++xx//PFZvHhxU5cCAAAAwGaiRVMO/uyzz+bWW29Nz549G6z/xje+kUcffTT3339/KioqMnz48Hz5y1/Or3/966YsBwAAijGqotQVlM6omlJXAMAWosnulFq2bFlOOumk3Hbbbdlhhx3q19fU1GTixIm54YYb8vnPfz4HHnhgJk2alKeeeiozZ85c51i1tbVZunRpgwYAAADAlqvJQqlhw4bl6KOPTr9+/RqsnzVrVlatWtVgfbdu3bLbbrtlxowZ6xxr9OjRqaioqG9VVVVNVTYAAAAABWiSUOqee+7J888/n9GjR6+1rbq6Oq1atUq7du0arO/QoUOqq6vXOd7IkSNTU1NT3xYuXNgUZQMAAABQkEZ/p9TChQtz7rnn5vHHH0/r1q0bZczy8vKUl5c3ylgAAAAAlF6j3yk1a9asvPnmm/n0pz+dFi1apEWLFpk+fXpuuummtGjRIh06dMjKlSuzZMmSBvstXrw4lZWVjV0OAAAAAJuhRr9Tqm/fvpkzZ06Ddaeeemq6deuWiy66KFVVVWnZsmWmTp2a448/Pkkyd+7cLFiwIL17927scgAAAADYDDX6nVJt2rRJ9+7dG7TtttsuO+20U7p3756KioqcfvrpGTFiRH71q19l1qxZOfXUU9O7d+8ccsghjV0OAMBmq3PnzikrK1urDRs2bJ39J0+evFbfxnpdAgBA0Rr9TqkPY8yYMWnWrFmOP/741NbWpn///rn55ptLUQoAQMk8++yzWb16df3yCy+8kH/5l3/JCSecsN592rZtm7lz59Yvl5WVNWmNAABNpZBQatq0aQ2WW7dunfHjx2f8+PFFHB4AYLPUvn37BsvXXHNNPvnJT+aII45Y7z5lZWXewwkAbBUa/fE9AAA23sqVK/PDH/4wp5122gbvflq2bFl23333VFVV5dhjj83vf//7Dxy7trY2S5cubdAAAEpNKAUAsBl46KGHsmTJkpxyyinr7dO1a9fcfvvtefjhh/PDH/4wa9asyaGHHprXX399g2OPHj06FRUV9a2qqqqRqwcA2HhCKQCAzcDEiRMzcODAdOrUab19evfunSFDhmT//ffPEUcckQceeCDt27fPrbfeusGxR44cmZqamvq2cOHCxi4fAGCjleRF5wAA/K/XXnstv/jFL/LAAw9s1H4tW7bMAQcckFdeeWWD/crLy1NeXr4pJQIANDqhFABAiU2aNCm77LJLjj766I3ab/Xq1ZkzZ06OOuqoJqoM+EedL3601CWUzPzWpa4A2Bp5fA8AoITWrFmTSZMmZejQoWnRouHnhUOGDMnIkSPrl6+88sr8/Oc/z5/+9Kc8//zzOfnkk/Paa6/ljDPOKLpsAIBN5k4pAGgKoypKXcHG6bJbqSv42PrFL36RBQsW5LTTTltr24IFC9Ks2f9+hvjuu+/mzDPPTHV1dXbYYYcceOCBeeqpp7LPPvsUWTIAQKMQSgEAlNAXvvCF1NXVrXPbtGnTGiyPGTMmY8aMKaAqAICm5/E9AAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAIASGTVqVMrKyhq0bt26bXCf+++/P926dUvr1q3To0ePPPbYYwVVCwDQuIRSAAAltO+++2bRokX17cknn1xv36eeeiqDBw/O6aefnt/85jcZNGhQBg0alBdeeKHAigEAGodQCgCghFq0aJHKysr6tvPOO6+374033pgBAwbkggsuyN57752rrroqn/70pzNu3LgCKwYAaBxCKQCAEnr55ZfTqVOn7LHHHjnppJOyYMGC9fadMWNG+vXr12Bd//79M2PGjA0eo7a2NkuXLm3QAABKTSgFAFAivXr1yuTJkzNlypRMmDAh8+bNy2c/+9n85S9/WWf/6urqdOjQocG6Dh06pLq6eoPHGT16dCoqKupbVVVVo50DAMBHJZQCACiRgQMH5oQTTkjPnj3Tv3//PPbYY1myZEnuu+++Rj3OyJEjU1NTU98WLlzYqOMDAHwULUpdAAAAf9euXbt86lOfyiuvvLLO7ZWVlVm8eHGDdYsXL05lZeUGxy0vL095eXmj1QkA0BjcKQUAsJlYtmxZXn311XTs2HGd23v37p2pU6c2WPf444+nd+/eRZQHANCohFIAACXyzW9+M9OnT8/8+fPz1FNP5bjjjkvz5s0zePDgJMmQIUMycuTI+v7nnntupkyZkuuvvz4vvfRSRo0aleeeey7Dhw8v1SkAAHxkHt8DACiR119/PYMHD84777yT9u3b5/DDD8/MmTPTvn37JMmCBQvSrNn/foZ46KGH5q677sqll16aSy65JHvttVceeuihdO/evVSnAADwkQmlAABK5J577tng9mnTpq217oQTTsgJJ5zQRBUBABTH43sAAAAAFE4oBQAAAEDhhFIAAAAAFE4oBQAAAEDhhFIAAAAAFE4oBQAAAEDhhFIAAAAAFE4oBQAAAEDhhFIAAAAAFE4oBQAAAEDhhFIAAAAAFE4oBQAAAEDhGj2UGj16dA466KC0adMmu+yySwYNGpS5c+c26LNixYoMGzYsO+20U7bffvscf/zxWbx4cWOXAgAAAMBmqtFDqenTp2fYsGGZOXNmHn/88axatSpf+MIXsnz58vo+3/jGN/Jf//Vfuf/++zN9+vS88cYb+fKXv9zYpQAAAACwmWrR2ANOmTKlwfLkyZOzyy67ZNasWfnc5z6XmpqaTJw4MXfddVc+//nPJ0kmTZqUvffeOzNnzswhhxyy1pi1tbWpra2tX166dGljlw0AAABAgZr8nVI1NTVJkh133DFJMmvWrKxatSr9+vWr79OtW7fstttumTFjxjrHGD16dCoqKupbVVVVU5cNAAAAQBNq0lBqzZo1Oe+883LYYYele/fuSZLq6uq0atUq7dq1a9C3Q4cOqa6uXuc4I0eOTE1NTX1buHBhU5YNAAAAQBNr9Mf3/tGwYcPywgsv5Mknn9ykccrLy1NeXt5IVQEAAABQak12p9Tw4cPzyCOP5Fe/+lU+8YlP1K+vrKzMypUrs2TJkgb9Fy9enMrKyqYqBwAAAIDNSKOHUnV1dRk+fHgefPDB/PKXv0yXLl0abD/wwAPTsmXLTJ06tX7d3Llzs2DBgvTu3buxywEAAABgM9Toj+8NGzYsd911Vx5++OG0adOm/j1RFRUV2WabbVJRUZHTTz89I0aMyI477pi2bdvmnHPOSe/evdf5zXsAAAAAbH0aPZSaMGFCkqRPnz4N1k+aNCmnnHJKkmTMmDFp1qxZjj/++NTW1qZ///65+eabG7sUAAAAADZTjR5K1dXVfWCf1q1bZ/z48Rk/fnxjHx4AAACALUCTvegcAAAAANZHKAUAAABA4YRSAAAAABROKAUAAABA4YRSAAAAABROKAUAAABA4YRSAAAAABROKAUAAABA4YRSAAAAABROKAUAAABA4YRSAAAAABROKAUAAABA4YRSAAAAABROKAUAAABA4YRSAAAAABROKAUAAABA4YRSAAAAABROKAUAAABA4YRSAAAAABROKAUAAABA4YRSAAAlMnr06Bx00EFp06ZNdtlllwwaNChz587d4D6TJ09OWVlZg9a6deuCKgYAaDxCKQCAEpk+fXqGDRuWmTNn5vHHH8+qVavyhS98IcuXL9/gfm3bts2iRYvq22uvvVZQxQAAjadFqQsAAPi4mjJlSoPlyZMnZ5dddsmsWbPyuc99br37lZWVpbKy8kMfp7a2NrW1tfXLS5cu3fhiAQAamTulAAA2EzU1NUmSHXfccYP9li1blt133z1VVVU59thj8/vf/36D/UePHp2Kior6VlVV1Wg1AwB8VEIpAIDNwJo1a3LeeeflsMMOS/fu3dfbr2vXrrn99tvz8MMP54c//GHWrFmTQw89NK+//vp69xk5cmRqamrq28KFC5viFAAANorH9wAANgPDhg3LCy+8kCeffHKD/Xr37p3evXvXLx966KHZe++9c+utt+aqq65a5z7l5eUpLy9v1HoBADaVUAoAoMSGDx+eRx55JE888UQ+8YlPbNS+LVu2zAEHHJBXXnmliaoDAGgaHt8DACiRurq6DB8+PA8++GB++ctfpkuXLhs9xurVqzNnzpx07NixCSoEAGg67pQCACiRYcOG5a677srDDz+cNm3apLq6OklSUVGRbbbZJkkyZMiQ7Lrrrhk9enSS5Morr8whhxySPffcM0uWLMl3v/vdvPbaaznjjDNKdh4AAB+FUAoAoEQmTJiQJOnTp0+D9ZMmTcopp5ySJFmwYEGaNfvfm9vffffdnHnmmamurs4OO+yQAw88ME899VT22WefosoGAGgUQikAgBKpq6v7wD7Tpk1rsDxmzJiMGTOmiSoCACiOd0oBAAAAUDihFAAAAACFE0oBAAAAUDihFAAAAACFE0oBAAAAUDihFAAAAACFE0oBAAAAUDihFAAAAACFE0oBAAAAUDihFAAAAACFE0oBAAAAUDihFAAAAACFE0oBAAAAUDihFAAAAACFE0oBAAAAUDihFAAAAACFE0oBAAAAUDihFAAAAACFE0oBAAAAUDihFAAAAACFE0oBAAAAUDihFAAAAACFE0oBAAAAUDihFAAAAACFE0oBAAAAUDihFAAAAACFE0oBAAAAUDihFAAAAACFE0oBAAAAUDihFAAAAACFE0oBAAAAUDihFAAAAACFE0oBAAAAUDihFAAAAACFE0oBAAAAUDihFAAAAACFE0oBAAAAUDihFAAAAACFE0oBAAAAUDihFAAAAACFE0oBAAAAUDihFAAAAACFK1koNX78+HTu3DmtW7dOr1698swzz5SqFACAktrY66L7778/3bp1S+vWrdOjR4889thjBVUKANB4ShJK3XvvvRkxYkSuuOKKPP/889lvv/3Sv3//vPnmm6UoBwCgZDb2uuipp57K4MGDc/rpp+c3v/lNBg0alEGDBuWFF14ouHIAgE3TohQHveGGG3LmmWfm1FNPTZLccsstefTRR3P77bfn4osvXqt/bW1tamtr65dramqSJEuXLi2mYNiCran9a6lL2ChLy+pKXcJGW/231aUuYaNtaT8/t7R5nGx5c9k8XvfYdXVN//e4sddFN954YwYMGJALLrggSXLVVVfl8ccfz7hx43LLLbes8xiluJbaEv+/bUxb2s+ARrWF/Y7ZGB/nef2xntPJVjuvP85zOvmYz+smntMf+lqqrmC1tbV1zZs3r3vwwQcbrB8yZEjdl770pXXuc8UVV9Ql0TRN0zRNK7QtXLhws7suqqqqqhszZkyDdZdffnldz54913sc11KapmmappWifdC1VOF3Sr399ttZvXp1OnTo0GB9hw4d8tJLL61zn5EjR2bEiBH1y2vWrMn//M//ZKeddkpZWVmT1tvYli5dmqqqqixcuDBt27YtdTnwkZjHbA3MYzakrq4uf/nLX9KpU6cmPc5HuS6qrq5eZ//q6ur1HmdrupbaEvj5wtbGnGZrZF43rQ97LVWSx/c2Vnl5ecrLyxusa9euXWmKaSRt27Y18dnimcdsDcxj1qeioqLUJTSarfFaakvg5wtbG3OarZF53XQ+zLVU4S8633nnndO8efMsXry4wfrFixensrKy6HIAAErmo1wXVVZWuo4CALYKhYdSrVq1yoEHHpipU6fWr1uzZk2mTp2a3r17F10OAEDJfJTrot69ezfonySPP/646ygAYItTksf3RowYkaFDh+Yzn/lMDj744IwdOzbLly+v/9aZrVl5eXmuuOKKtW6hhy2JeczWwDxmc/FB10VDhgzJrrvumtGjRydJzj333BxxxBG5/vrrc/TRR+eee+7Jc889l+9///ulPA3+gZ8vbG3MabZG5vXmoayuroDvOl6HcePG5bvf/W6qq6uz//7756abbkqvXr1KUQoAQElt6LqoT58+6dy5cyZPnlzf//7778+ll16a+fPnZ6+99sp1112Xo446qkTVAwB8NCULpQAAAAD4+Cr8nVIAAAAAIJQCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCqQ/plFNOSVlZWcrKytKyZct06dIlF154YVasWNGg3yOPPJIjjjgibdq0ybbbbpuDDjqowbflJMn8+fNTVlaW5s2b589//nODbYsWLUqLFi1SVlaW+fPnN+i/rjZz5symPG22Mu/P46997WtrbRs2bFjKyspyyimnNOj7z23AgAH1+3Tu3Hmd8/C8885Lnz596pdHjRpVv3+LFi2y884753Of+1zGjh2b2traJjlXOOWUUzJo0KC1/vt9P/7xj9O6detcf/319X0+aM4DW6/Ro0fnoIMOSps2bbLLLrtk0KBBmTt37gb3GTVqVPbff/+11r9/7TZ79uymKRY+pAkTJqRnz55p27Zt2rZtm969e+enP/3pBvcxr9mSXHPNNSkrK8t55523wX7m9eZLKLURBgwYkEWLFuVPf/pTxowZk1tvvTVXXHFF/fbvfe97OfbYY3PYYYfl6aefzu9+97uceOKJ+drXvpZvfvOba42366675s4772yw7o477siuu+66zuP/4he/yKJFixq0Aw88sHFPkq1eVVVV7rnnnvztb3+rX7dixYrcdddd2W233Rr0fX/O/2O7++67G/Rp3bp1Lrroog887r777ptFixZlwYIF+dWvfpUTTjgho0ePzqGHHpq//OUvjXNy8CH94Ac/yEknnZQJEybk/PPPr1//YeY8sHWaPn16hg0blpkzZ+bxxx/PqlWr8oUvfCHLly8vdWnwkX3iE5/INddck1mzZuW5557L5z//+Rx77LH5/e9/X+rSYJM9++yzufXWW9OzZ89Sl8ImEEpthPLy8lRWVqaqqiqDBg1Kv3798vjjjydJFi5cmPPPPz/nnXdevvOd72SfffbJnnvumfPPPz/f/e53c/311+fpp59uMN7QoUMzadKkBusmTZqUoUOHrvP4O+20UyorKxu0li1bNs3JstX69Kc/naqqqjzwwAP16x544IHstttuOeCAAxr0fX/O/2PbYYcdGvQ566yzMnPmzDz22GMbPG6LFi1SWVmZTp06pUePHjnnnHMyffr0vPDCC7n22msb7wThA1x33XU555xzcs899+TUU09tsO3DzHlg6zRlypSccsop2XfffbPffvtl8uTJWbBgQWbNmtUo40+fPj0HH3xwysvL07Fjx1x88cV577336rf36dMnw4cPz/Dhw1NRUZGdd945l112Werq6hrl+Hw8HXPMMTnqqKOy11575VOf+lS+/e1vZ/vtt2+0py3Ma0pl2bJlOemkk3Lbbbc1+rWaeV0sodRH9MILL+Spp55Kq1atkvz9MZBVq1at846or371q9l+++3X+rT9S1/6Ut599908+eSTSZInn3wy7777bo455pimPwE+1k477bQGgejtt9++1j/OP6wuXbrka1/7WkaOHJk1a9Zs1L7dunXLwIEDGwRk0JQuuuiiXHXVVXnkkUdy3HHHlbocYDNWU1OTJNlxxx03eaw///nPOeqoo3LQQQflt7/9bSZMmJCJEyfm6quvbtDvjjvuSIsWLfLMM8/kxhtvzA033JAf/OAHm3x8SJLVq1fnnnvuyfLly9O7d+9NHs+8ppSGDRuWo48+Ov369WvUcc3r4rUodQFbkkceeSTbb7993nvvvdTW1qZZs2YZN25ckuSPf/xjKioq0rFjx7X2a9WqVfbYY4/88Y9/bLC+ZcuWOfnkk3P77bfn8MMPz+23356TTz55vXc/HXrooWnWrGGOuGzZskY6Oz5OTj755IwcOTKvvfZakuTXv/517rnnnkybNq1Bv/fn/D+65JJLcskllzRYd+mll2bSpEn50Y9+lP/7f//vRtXSrVu3/PznP9/4k4CN9NOf/jQPP/xwpk6dms9//vPr7PNh5zywdVuzZk3OO++8HHbYYenevfsG+86ZM2etnxv//Gn5zTffnKqqqowbNy5lZWXp1q1b3njjjVx00UW5/PLL66/vqqqqMmbMmJSVlaVr166ZM2dOxowZkzPPPLNxT5CPlTlz5qR3795ZsWJFtt9++zz44IPZZ599PnAf85rN1T333JPnn38+zz777EbtZ15vnoRSG+HII4/MhAkTsnz58owZMyYtWrTI8ccfv0ljnnbaaTn00EPzne98J/fff39mzJjR4NbAf3Tvvfdm77333qTjQZK0b98+Rx99dCZPnpy6urocffTR2Xnnndfq9/6c/0fr+sS4ffv2+eY3v5nLL788X/nKVzaqlrq6upSVlW3cCcBH0LNnz7z99tu54oorcvDBB691UZJ8+DkPbN2GDRuWF154of5u9g3p2rVrfvKTnzRY9+c//7nBF368+OKL6d27d4Pfd4cddliWLVuW119/vf6djoccckiDPr17987111+f1atXp3nz5pt4Vnxcde3aNbNnz05NTU1+/OMfZ+jQoZk+ffoGgynzms3VwoULc+655+bxxx9P69atN2pf83rzJJTaCNttt1323HPPJH9/3Gm//fbLxIkTc/rpp+dTn/pUampq8sYbb6RTp04N9lu5cmVeffXVHHnkkWuN2aNHj3Tr1i2DBw/O3nvvne7du6/3zf9VVVX1x4dNddppp2X48OFJkvHjx6+zzz/O+Q8yYsSI3Hzzzbn55ps3qo4XX3wxXbp02ah94KPYdddd8+Mf/zhHHnlkBgwYkJ/+9Kdp06ZNgz4bM+eBrdPw4cPzyCOP5IknnsgnPvGJD+zfqlWrtX5utGjhEpvNxz/O0QMPPDDPPvtsbrzxxtx6660fap/3mddsDmbNmpU333wzn/70p+vXrV69Ok888UTGjRuX2tra9YZC5vXmyTulPqJmzZrlkksuyaWXXpq//e1vOf7449OyZcv6rxb/R7fcckuWL1+ewYMHr3Os0047LdOmTctpp53W1GVDvQEDBmTlypVZtWpV+vfvv8njbb/99rnsssvy7W9/+0N/m95LL72UKVOmbPIdh/Bh7b777pk+fXqqq6szYMAA3/wI1Kurq8vw4cPz4IMP5pe//GWjfmCy9957Z8aMGQ0eE/n1r3+dNm3aNAi+/vlLcWbOnJm99trLp+40qjVr1qS2tnaTxzGvKYW+fftmzpw5mT17dn37zGc+k5NOOimzZ8/e5HllXhdPKLUJTjjhhDRv3jzjx4/Pbrvtluuuuy5jx47N//t//y8vvfRSXn311dxwww258MILc/7556dXr17rHOfMM8/MW2+9lTPOOGODx3vnnXdSXV3doK1YsaIpTo2PgebNm+fFF1/MH/7wh/X+8KytrV1rzr399tvrHfOss85KRUVF7rrrrrW2vffee6murs4bb7yROXPm5Hvf+16OOOKI7L///rngggsa7bzgg1RVVWXatGl58803079//yxdurR+28bOeWDrMWzYsPzwhz/MXXfdlTZt2tT/DPjb3/62yWN//etfz8KFC3POOefkpZdeysMPP5wrrrgiI0aMaPC+0AULFmTEiBGZO3du7r777nzve9/Lueeeu8nH5+Nr5MiReeKJJzJ//vzMmTMnI0eOzLRp03LSSSdt8tjmNaXQpk2bdO/evUHbbrvtstNOO33gOwA/DPO6eO5V2wQtWrTI8OHDc9111+Xss8/Oeeedlz322CP/8R//kRtvvDGrV6/OvvvumwkTJmzwm81atGixzvf5/LN1fbPA3XffnRNPPHGTzoOPr7Zt225w+5QpU9Z6eX/Xrl3z0ksvrbN/y5Ytc9VVV+Xf/u3f1tr2+9//Ph07dkzz5s1TUVGRffbZJyNHjszZZ5+d8vLyj34S8BF84hOfyLRp03LkkUemf//++dnPfpZk4+c8sPV4/31y//hukSSZNGlSTjnllE0ae9ddd81jjz2WCy64IPvtt1923HHHnH766bn00ksb9BsyZEj+9re/5eCDD07z5s1z7rnn5qyzztqkY/Px9uabb2bIkCFZtGhRKioq0rNnz/zsZz/Lv/zLv2zy2OY1WyPzunhldf/8unkAAKBQffr0yf7775+xY8eWuhRoNOY1WyPzunF5fA8AAACAwgmlAAAAACicx/cAAAAAKJw7pQAAAAAonFAKAAAAgMIJpQAAAAAonFAKAAAAgMIJpQAAAAAonFAKAAAAgMIJpQAAAAAonFAKAAAAgML9f0mK8Rcu8KvcAAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ],
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# Data from the tables\n",
        "models = ['ROME', 'MEND', 'IKE']\n",
        "\n",
        "# GPT-2-xl results\n",
        "gpt2_generalization = [96.5, 58.3, 95.4]\n",
        "gpt2_locality = [75.41, 44.88, 76.7]\n",
        "gpt2_accuracy = [100, 94.2, 100]\n",
        "\n",
        "# Llama-2 7B results\n",
        "llama_generalization = [87.04, 90.27, 99.4]\n",
        "llama_locality = [99.36, 97.02, 69.2]\n",
        "llama_accuracy = [99.42, 94.24, 100]\n",
        "\n",
        "# T5 3B results\n",
        "t5_generalization = [0, 93.27, 83.2]\n",
        "t5_locality = [0, 91.85, 36.67]\n",
        "t5_accuracy = [0, 81.40, 97.7]\n",
        "\n",
        "# Portability results for ROME\n",
        "portability_pre_edit = [19.33, 7.33, 10.67]\n",
        "portability_post_edit = [8.0, 8.67, 10.0]\n",
        "portability_labels = ['2 Hop', '3 Hop', '4 Hop']\n",
        "\n",
        "# Function to plot bar graphs\n",
        "def plot_multiple_bar_subplots(generalization_data, locality_data, accuracy_data, portability_pre_edit, portability_post_edit, portability_labels):\n",
        "    fig, axs = plt.subplots(2, 2, figsize=(12, 10))\n",
        "\n",
        "    # GPT-2-xl subplot\n",
        "    axs[0, 0].bar(models, generalization_data, width=0.25, label='Generalization')\n",
        "    axs[0, 0].bar(np.arange(len(models)) + 0.25, locality_data, width=0.25, label='Locality')\n",
        "    axs[0, 0].bar(np.arange(len(models)) + 0.5, accuracy_data, width=0.25, label='Accuracy')\n",
        "    axs[0, 0].set_title('GPT-2-xl')\n",
        "    axs[0, 0].legend()\n",
        "\n",
        "    # Llama-2 7B subplot\n",
        "    axs[0, 1].bar(models, llama_generalization, width=0.25, label='Generalization')\n",
        "    axs[0, 1].bar(np.arange(len(models)) + 0.25, llama_locality, width=0.25, label='Locality')\n",
        "    axs[0, 1].bar(np.arange(len(models)) + 0.5, llama_accuracy, width=0.25, label='Accuracy')\n",
        "    axs[0, 1].set_title('Llama-2 7B')\n",
        "    axs[0, 1].legend()\n",
        "\n",
        "    # T5 3B subplot\n",
        "    axs[1, 0].bar(models, t5_generalization, width=0.25, label='Generalization')\n",
        "    axs[1, 0].bar(np.arange(len(models)) + 0.25, t5_locality, width=0.25, label='Locality')\n",
        "    axs[1, 0].bar(np.arange(len(models)) + 0.5, t5_accuracy, width=0.25, label='Accuracy')\n",
        "    axs[1, 0].set_title('T5 3B')\n",
        "    axs[1, 0].legend()\n",
        "\n",
        "    # Portability subplot\n",
        "    x = np.arange(len(portability_labels))\n",
        "    width = 0.35\n",
        "    axs[1, 1].bar(x - width/2, portability_pre_edit, width, label='Pre Edit')\n",
        "    axs[1, 1].bar(x + width/2, portability_post_edit, width, label='Post Edit')\n",
        "    axs[1, 1].set_title('Portability for ROME')\n",
        "    axs[1, 1].set_xticks(x)\n",
        "    axs[1, 1].set_xticklabels(portability_labels)\n",
        "    axs[1, 1].legend()\n",
        "\n",
        "    plt.tight_layout()\n",
        "    plt.show()\n",
        "\n",
        "# Plotting subplots\n",
        "plot_multiple_bar_subplots(gpt2_generalization, gpt2_locality, gpt2_accuracy, portability_pre_edit, portability_post_edit, portability_labels)\n"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.11.7"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "ecb1b3f80a3749ca9b1375137c21c558": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2c9f1b8a3a0c434991abe5e74577f6e7",
              "IPY_MODEL_7f9533b28ffd416998f7c8216cb7eeed",
              "IPY_MODEL_b4169506de3445eb9219703302d073ed"
            ],
            "layout": "IPY_MODEL_ee67624927d0439c853170fe50f5bbc8"
          }
        },
        "2c9f1b8a3a0c434991abe5e74577f6e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a91fa038e7874904801346047ac202ff",
            "placeholder": "​",
            "style": "IPY_MODEL_53c7622310124578a9d3102259079b45",
            "value": "config.json: 100%"
          }
        },
        "7f9533b28ffd416998f7c8216cb7eeed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fca68c60fa6c482cbe64929484cc589d",
            "max": 689,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c42bce8bf71040e7995ba0dc1c9f7456",
            "value": 689
          }
        },
        "b4169506de3445eb9219703302d073ed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e31f15e0bcd6418dac8e249101a8f573",
            "placeholder": "​",
            "style": "IPY_MODEL_91ab434064184d289db882c24e86af6a",
            "value": " 689/689 [00:00&lt;00:00, 50.9kB/s]"
          }
        },
        "ee67624927d0439c853170fe50f5bbc8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a91fa038e7874904801346047ac202ff": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "53c7622310124578a9d3102259079b45": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fca68c60fa6c482cbe64929484cc589d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c42bce8bf71040e7995ba0dc1c9f7456": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e31f15e0bcd6418dac8e249101a8f573": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "91ab434064184d289db882c24e86af6a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5f5f0ff6c8764fd6a671b9bd06546cff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_39a6b3c12cca41b7a53df90e6d882538",
              "IPY_MODEL_a4b7c7a3fd9444d29a635129feff49d0",
              "IPY_MODEL_b829570cdbff4ae483cb737fd1704275"
            ],
            "layout": "IPY_MODEL_147fd574b00e42f59ba185cd89cf938f"
          }
        },
        "39a6b3c12cca41b7a53df90e6d882538": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_72c18038466846beb5204683e4438950",
            "placeholder": "​",
            "style": "IPY_MODEL_0e6d52706330472282c511ff150cbb97",
            "value": "model.safetensors: 100%"
          }
        },
        "a4b7c7a3fd9444d29a635129feff49d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e12d7927ba5440208bd031473d5cb780",
            "max": 6431829964,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b955134a9b9f44578e36d2feac47d114",
            "value": 6431829964
          }
        },
        "b829570cdbff4ae483cb737fd1704275": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6c1401242cbe44e1ba6e30d99ac2bb7f",
            "placeholder": "​",
            "style": "IPY_MODEL_c5a932fd6eb24b6cbb09e3c03f07ce89",
            "value": " 6.43G/6.43G [00:50&lt;00:00, 201MB/s]"
          }
        },
        "147fd574b00e42f59ba185cd89cf938f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "72c18038466846beb5204683e4438950": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0e6d52706330472282c511ff150cbb97": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e12d7927ba5440208bd031473d5cb780": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b955134a9b9f44578e36d2feac47d114": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6c1401242cbe44e1ba6e30d99ac2bb7f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c5a932fd6eb24b6cbb09e3c03f07ce89": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b5e0b5d99b3e48f58520c07f31474ff5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7d9c5f83c2354d8cb85bc9765a9219d1",
              "IPY_MODEL_0135b84b3d804180b7ae0cd464a12cd6",
              "IPY_MODEL_04fcf99977ca42238d736bb8f9962199"
            ],
            "layout": "IPY_MODEL_dd798af4c9774ae8a3b5c93f0c466180"
          }
        },
        "7d9c5f83c2354d8cb85bc9765a9219d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1f258cdd901e4d8cbd4328a87ccebeee",
            "placeholder": "​",
            "style": "IPY_MODEL_d849f04206cb42f08660f6200628aa5a",
            "value": "generation_config.json: 100%"
          }
        },
        "0135b84b3d804180b7ae0cd464a12cd6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6ecee932134d4a4ea13ff6ecd1f2bb93",
            "max": 124,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_cf95a2dc546f42c99d06ebd750cd2838",
            "value": 124
          }
        },
        "04fcf99977ca42238d736bb8f9962199": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_84f2a4694183437dae96e414b4ca036a",
            "placeholder": "​",
            "style": "IPY_MODEL_606433a460e14a0492eb0cf7113f06ff",
            "value": " 124/124 [00:00&lt;00:00, 7.83kB/s]"
          }
        },
        "dd798af4c9774ae8a3b5c93f0c466180": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1f258cdd901e4d8cbd4328a87ccebeee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d849f04206cb42f08660f6200628aa5a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6ecee932134d4a4ea13ff6ecd1f2bb93": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cf95a2dc546f42c99d06ebd750cd2838": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "84f2a4694183437dae96e414b4ca036a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "606433a460e14a0492eb0cf7113f06ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e84b28f5eeb14cd8a5e4679539c4c0a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_cc4a672362384a65a671b65474522b6f",
              "IPY_MODEL_94f3261b04cb4402bd7f4a6586e6a6c7",
              "IPY_MODEL_eee70d31e0bd4825b58548037c0140ef"
            ],
            "layout": "IPY_MODEL_c78edfafed6b4baeaaae1562609d2ba1"
          }
        },
        "cc4a672362384a65a671b65474522b6f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_eb53cd4e45984671b1d5592ff05c7e20",
            "placeholder": "​",
            "style": "IPY_MODEL_0963509938314d4487f3fd64b25c22aa",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "94f3261b04cb4402bd7f4a6586e6a6c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3a81748e530c42e3b3cf3ce38769d057",
            "max": 26,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_23d4e8b354b040faa92ef87df33f27a6",
            "value": 26
          }
        },
        "eee70d31e0bd4825b58548037c0140ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1124dbbf961848d6a35b602e8d306bf8",
            "placeholder": "​",
            "style": "IPY_MODEL_4c22806f41b6458a856b2d1b37a1c9a9",
            "value": " 26.0/26.0 [00:00&lt;00:00, 1.87kB/s]"
          }
        },
        "c78edfafed6b4baeaaae1562609d2ba1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "eb53cd4e45984671b1d5592ff05c7e20": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0963509938314d4487f3fd64b25c22aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3a81748e530c42e3b3cf3ce38769d057": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "23d4e8b354b040faa92ef87df33f27a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1124dbbf961848d6a35b602e8d306bf8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4c22806f41b6458a856b2d1b37a1c9a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ea313873c85e4d149457cff89998bb8c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f52b3313e7574fb2904674f4e40e9cd2",
              "IPY_MODEL_cc4aaf17191d479fa94f3143aa588183",
              "IPY_MODEL_2ce91981bd33400480ce5765ffca84da"
            ],
            "layout": "IPY_MODEL_1a74bd9840504e18a4a76d969f0ed80c"
          }
        },
        "f52b3313e7574fb2904674f4e40e9cd2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bde2c018f6b54961b67a3ba8f26fe440",
            "placeholder": "​",
            "style": "IPY_MODEL_8d42c6b17bf544d8bb20921aa974bbfb",
            "value": "vocab.json: 100%"
          }
        },
        "cc4aaf17191d479fa94f3143aa588183": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6eb0c026617a4fd9b1812362d4ecf578",
            "max": 1042301,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8bba8207a5d949e399584c95632717de",
            "value": 1042301
          }
        },
        "2ce91981bd33400480ce5765ffca84da": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ad05004d32914e4dac625f4be8ad6dc1",
            "placeholder": "​",
            "style": "IPY_MODEL_ec9332ad95f84f008559889e172e3977",
            "value": " 1.04M/1.04M [00:00&lt;00:00, 6.67MB/s]"
          }
        },
        "1a74bd9840504e18a4a76d969f0ed80c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bde2c018f6b54961b67a3ba8f26fe440": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8d42c6b17bf544d8bb20921aa974bbfb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6eb0c026617a4fd9b1812362d4ecf578": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8bba8207a5d949e399584c95632717de": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ad05004d32914e4dac625f4be8ad6dc1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ec9332ad95f84f008559889e172e3977": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "dae5711258244feca861e5d4351ee430": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2b562aad74d7409589dd03b771e29325",
              "IPY_MODEL_5ed562607ebc49f08ea44370918d4c81",
              "IPY_MODEL_c0c8b47c44d7462586ef8edfe2ed431c"
            ],
            "layout": "IPY_MODEL_41fe6c0e0a98455b9bf354a728e900af"
          }
        },
        "2b562aad74d7409589dd03b771e29325": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_df340caf93ff461a914b2b32bb3e76a9",
            "placeholder": "​",
            "style": "IPY_MODEL_397b99c827254c2385204d2d873916c1",
            "value": "merges.txt: 100%"
          }
        },
        "5ed562607ebc49f08ea44370918d4c81": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b720c3e53c3d43ccba0588babd207568",
            "max": 456318,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5d0347f9ce7d4ee8aced1553bc582ba4",
            "value": 456318
          }
        },
        "c0c8b47c44d7462586ef8edfe2ed431c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cb3c1bd8170a4e3ba0d2da8966d7a41c",
            "placeholder": "​",
            "style": "IPY_MODEL_62bd4b3359a14cad82e8c46450f39793",
            "value": " 456k/456k [00:00&lt;00:00, 31.4MB/s]"
          }
        },
        "41fe6c0e0a98455b9bf354a728e900af": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "df340caf93ff461a914b2b32bb3e76a9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "397b99c827254c2385204d2d873916c1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b720c3e53c3d43ccba0588babd207568": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5d0347f9ce7d4ee8aced1553bc582ba4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "cb3c1bd8170a4e3ba0d2da8966d7a41c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "62bd4b3359a14cad82e8c46450f39793": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f35a76966e0c40cdb772bce08659f353": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_98c222f87419461a91569159dc8baf56",
              "IPY_MODEL_a9c8171fb99840df87c1a07c5d6acff8",
              "IPY_MODEL_678cd67aa16a4bbda8cb106ee02b36d6"
            ],
            "layout": "IPY_MODEL_5d5e34679c3743db8a7f74df47cdf250"
          }
        },
        "98c222f87419461a91569159dc8baf56": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b12f726b4d054444890c3abb58e14fff",
            "placeholder": "​",
            "style": "IPY_MODEL_11a1b727dcab44378b81707bc44a142d",
            "value": "tokenizer.json: 100%"
          }
        },
        "a9c8171fb99840df87c1a07c5d6acff8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_52413c2ce2bd4e62adfee648d1c479d5",
            "max": 1355256,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_3818aaad4cc84e73832730f3bec84a63",
            "value": 1355256
          }
        },
        "678cd67aa16a4bbda8cb106ee02b36d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e522b33296a24ca2a07a58161c47ab41",
            "placeholder": "​",
            "style": "IPY_MODEL_8514592d7a69453eb8ce2fdfadedd243",
            "value": " 1.36M/1.36M [00:00&lt;00:00, 11.3MB/s]"
          }
        },
        "5d5e34679c3743db8a7f74df47cdf250": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b12f726b4d054444890c3abb58e14fff": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "11a1b727dcab44378b81707bc44a142d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "52413c2ce2bd4e62adfee648d1c479d5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3818aaad4cc84e73832730f3bec84a63": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e522b33296a24ca2a07a58161c47ab41": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8514592d7a69453eb8ce2fdfadedd243": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e11a74c041554bdb99fe85608e8826a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d5e4c5dbb28d4c7ba7f806fa8fdd34c1",
              "IPY_MODEL_7d22e9c498634c2f9c464faee374dd1a",
              "IPY_MODEL_8cde2c73c71e46759e137a72ab934011"
            ],
            "layout": "IPY_MODEL_8cef951d2aaf408bb60adaf0ff43f106"
          }
        },
        "d5e4c5dbb28d4c7ba7f806fa8fdd34c1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4222230c8cd540e1be9507c54236e705",
            "placeholder": "​",
            "style": "IPY_MODEL_50631dd33a8345fdb9206fba151b99b0",
            "value": "  0%"
          }
        },
        "7d22e9c498634c2f9c464faee374dd1a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "danger",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9134bedbd1b147d7ac5db5d9a4b74472",
            "max": 1000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_44a2a9e70ffe43a486a074530c2535ea",
            "value": 0
          }
        },
        "8cde2c73c71e46759e137a72ab934011": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9ade9b0fb3f74300b4aeca9c75704658",
            "placeholder": "​",
            "style": "IPY_MODEL_193cb87795a24e37a4785132b933501d",
            "value": " 0/1000 [00:00&lt;?, ?it/s]"
          }
        },
        "8cef951d2aaf408bb60adaf0ff43f106": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4222230c8cd540e1be9507c54236e705": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "50631dd33a8345fdb9206fba151b99b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9134bedbd1b147d7ac5db5d9a4b74472": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "44a2a9e70ffe43a486a074530c2535ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9ade9b0fb3f74300b4aeca9c75704658": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "193cb87795a24e37a4785132b933501d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}