Portability_4hop.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v_cGEnqs9An4",
        "outputId": "df4f1bd0-c07b-4d37-ac61-f82f1f938e13"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting datasets\n",
            "  Downloading datasets-2.18.0-py3-none-any.whl (510 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m510.5/510.5 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting accelerate\n",
            "  Downloading accelerate-0.29.2-py3-none-any.whl (297 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m297.4/297.4 kB\u001b[0m \u001b[31m9.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from datasets) (3.13.4)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from datasets) (1.25.2)\n",
            "Requirement already satisfied: pyarrow>=12.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (14.0.2)\n",
            "Requirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.10/dist-packages (from datasets) (0.6)\n",
            "Collecting dill<0.3.9,>=0.3.0 (from datasets)\n",
            "  Downloading dill-0.3.8-py3-none-any.whl (116 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m8.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (2.0.3)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (4.66.2)\n",
            "Collecting xxhash (from datasets)\n",
            "  Downloading xxhash-3.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m9.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting multiprocess (from datasets)\n",
            "  Downloading multiprocess-0.70.16-py310-none-any.whl (134 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m7.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: fsspec[http]<=2024.2.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2023.6.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.9.3)\n",
            "Requirement already satisfied: huggingface-hub>=0.19.4 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.20.3)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets) (24.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (6.0.1)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate) (5.9.5)\n",
            "Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (2.2.1+cu121)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from accelerate) (0.4.2)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->datasets) (4.11.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2024.2.2)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.1.3)\n",
            "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch>=1.10.0->accelerate)\n",
            "  Using cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
            "Collecting nvidia-cuda-runtime-cu12==12.1.105 (from torch>=1.10.0->accelerate)\n",
            "  Using cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
            "Collecting nvidia-cuda-cupti-cu12==12.1.105 (from torch>=1.10.0->accelerate)\n",
            "  Using cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
            "Collecting nvidia-cudnn-cu12==8.9.2.26 (from torch>=1.10.0->accelerate)\n",
            "  Using cached nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n",
            "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch>=1.10.0->accelerate)\n",
            "  Using cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
            "Collecting nvidia-cufft-cu12==11.0.2.54 (from torch>=1.10.0->accelerate)\n",
            "  Using cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
            "Collecting nvidia-curand-cu12==10.3.2.106 (from torch>=1.10.0->accelerate)\n",
            "  Using cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
            "Collecting nvidia-cusolver-cu12==11.4.5.107 (from torch>=1.10.0->accelerate)\n",
            "  Using cached nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
            "Collecting nvidia-cusparse-cu12==12.1.0.106 (from torch>=1.10.0->accelerate)\n",
            "  Using cached nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
            "Collecting nvidia-nccl-cu12==2.19.3 (from torch>=1.10.0->accelerate)\n",
            "  Using cached nvidia_nccl_cu12-2.19.3-py3-none-manylinux1_x86_64.whl (166.0 MB)\n",
            "Collecting nvidia-nvtx-cu12==12.1.105 (from torch>=1.10.0->accelerate)\n",
            "  Using cached nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
            "Requirement already satisfied: triton==2.2.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2.2.0)\n",
            "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.10.0->accelerate)\n",
            "  Using cached nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl (21.1 MB)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2023.4)\n",
            "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2024.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.5)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\n",
            "Installing collected packages: xxhash, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, dill, nvidia-cusparse-cu12, nvidia-cudnn-cu12, multiprocess, nvidia-cusolver-cu12, datasets, accelerate\n",
            "Successfully installed accelerate-0.29.2 datasets-2.18.0 dill-0.3.8 multiprocess-0.70.16 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.19.3 nvidia-nvjitlink-cu12-12.4.127 nvidia-nvtx-cu12-12.1.105 xxhash-3.4.1\n"
          ]
        }
      ],
      "source": [
        "!pip install datasets accelerate"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "22sQ_oMgEHR9"
      },
      "outputs": [],
      "source": [
        "%%bash\n",
        "!(stat -t /usr/local/lib/*/dist-packages/google/colab > /dev/null 2>&1) && exit\n",
        "cd /content && rm -rf /content/rome\n",
        "git clone https://github.com/kmeng01/rome rome > install.log 2>&1\n",
        "pip install -r /content/rome/scripts/colab_reqs/rome.txt >> install.log 2>&1\n",
        "pip install --upgrade google-cloud-storage >> install.log 2>&1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a3QSWiHz-LZv",
        "outputId": "9e5a8861-d616-4d34-b9cf-0d6f662c2bda"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2024-04-16 14:55:40--  https://rome.baulab.info/data/dsets/counterfact.json\n",
            "Resolving rome.baulab.info (rome.baulab.info)... 35.232.255.106\n",
            "Connecting to rome.baulab.info (rome.baulab.info)|35.232.255.106|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 45108470 (43M) [application/json]\n",
            "Saving to: ‘counterfact.json’\n",
            "\n",
            "counterfact.json    100%[===================>]  43.02M  50.2MB/s    in 0.9s    \n",
            "\n",
            "2024-04-16 14:55:41 (50.2 MB/s) - ‘counterfact.json’ saved [45108470/45108470]\n",
            "\n"
          ]
        }
      ],
      "source": [
        "!wget 'https://rome.baulab.info/data/dsets/counterfact.json' -O counterfact.json"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "39jdfGmjU9Lo",
        "outputId": "9bde6b39-538f-409e-d102-a50740c67dd1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting sentence_transformers\n",
            "  Downloading sentence_transformers-2.6.1-py3-none-any.whl (163 kB)\n",
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/163.3 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━\u001b[0m \u001b[32m143.4/163.3 kB\u001b[0m \u001b[31m4.1 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m163.3/163.3 kB\u001b[0m \u001b[31m3.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: transformers<5.0.0,>=4.32.0 in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (4.38.2)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (4.66.2)\n",
            "Requirement already satisfied: torch>=1.11.0 in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (2.2.1+cu121)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (1.25.2)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (1.2.2)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (1.11.4)\n",
            "Requirement already satisfied: huggingface-hub>=0.15.1 in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (0.20.3)\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (9.4.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence_transformers) (3.13.4)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence_transformers) (2023.6.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence_transformers) (2.31.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence_transformers) (6.0.1)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence_transformers) (4.11.0)\n",
            "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence_transformers) (24.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (3.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (3.1.3)\n",
            "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (8.9.2.26)\n",
            "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (12.1.3.1)\n",
            "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (11.0.2.54)\n",
            "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (10.3.2.106)\n",
            "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (11.4.5.107)\n",
            "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (12.1.0.106)\n",
            "Requirement already satisfied: nvidia-nccl-cu12==2.19.3 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (2.19.3)\n",
            "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (12.1.105)\n",
            "Requirement already satisfied: triton==2.2.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (2.2.0)\n",
            "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.10/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.11.0->sentence_transformers) (12.4.127)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.32.0->sentence_transformers) (2023.12.25)\n",
            "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.32.0->sentence_transformers) (0.15.2)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.32.0->sentence_transformers) (0.4.2)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->sentence_transformers) (1.4.0)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->sentence_transformers) (3.4.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.11.0->sentence_transformers) (2.1.5)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.15.1->sentence_transformers) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.15.1->sentence_transformers) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.15.1->sentence_transformers) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.15.1->sentence_transformers) (2024.2.2)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.11.0->sentence_transformers) (1.3.0)\n",
            "Installing collected packages: sentence_transformers\n",
            "Successfully installed sentence_transformers-2.6.1\n"
          ]
        }
      ],
      "source": [
        "!pip install sentence_transformers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PE5xt80KVJf4"
      },
      "outputs": [],
      "source": [
        "IS_COLAB = False\n",
        "ALL_DEPS = False\n",
        "try:\n",
        "    import google.colab, torch, os\n",
        "\n",
        "    IS_COLAB = True\n",
        "    os.chdir(\"/content/rome\")\n",
        "    if not torch.cuda.is_available():\n",
        "        raise Exception(\"Change runtime type to include a GPU.\")\n",
        "except ModuleNotFoundError as _:\n",
        "    pass"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "x4o712m6VL9y"
      },
      "outputs": [],
      "source": [
        "%load_ext autoreload\n",
        "%autoreload 2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KINgP9of80mM"
      },
      "outputs": [],
      "source": [
        "from sentence_transformers import SentenceTransformer\n",
        "import pickle\n",
        "import json\n",
        "import torch\n",
        "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
        "\n",
        "from util import nethook\n",
        "from util.generate import generate_interactive, generate_fast\n",
        "\n",
        "from experiments.py.demo import demo_model_editing, stop_execution"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7JJOCxzA9npu"
      },
      "outputs": [],
      "source": [
        "data_path = '/content/counterfact.json'\n",
        "with open(data_path, 'r') as f:\n",
        "    lines = json.load(f)\n",
        "\n",
        "for i, line in enumerate(lines):\n",
        "    subject = line['requested_rewrite']['subject']\n",
        "    prompts = line['paraphrase_prompts']\n",
        "    new_prompts = []\n",
        "    for prompt in prompts:\n",
        "        prefix = prompt[:prompt.find(subject)]\n",
        "        while '.' in prefix:\n",
        "            prefix = prefix[prefix.find('.')+1:]\n",
        "        while '\\n' in prefix:\n",
        "            prefix = prefix[prefix.find('\\n')+1:]\n",
        "        new_prompt = (prefix+prompt[prompt.find(subject):]).strip()\n",
        "        if 'Category' in new_prompt:\n",
        "            new_prompt = new_prompt[new_prompt.find(subject):]\n",
        "        new_prompts.append(new_prompt)\n",
        "        #print(new_prompt)\n",
        "    lines[i]['paraphrase_prompts'] = new_prompts\n",
        "\n",
        "with open(data_path, 'w') as f:\n",
        "    json.dump(lines, f, indent=2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1TQNvPWsA9gL"
      },
      "outputs": [],
      "source": [
        "data={}\n",
        "with open(data_path, \"r\") as file:\n",
        "    data = json.load(file)\n",
        "\n",
        "train_data =  data[:2000]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pFoJe-VTDdMS",
        "outputId": "93cd5e54-9764-4e6c-ffbe-397169b21c18"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[{'case_id': 0,\n",
              "  'pararel_idx': 2796,\n",
              "  'requested_rewrite': {'prompt': 'The mother tongue of {} is',\n",
              "   'relation_id': 'P103',\n",
              "   'target_new': {'str': 'English', 'id': 'Q1860'},\n",
              "   'target_true': {'str': 'French', 'id': 'Q150'},\n",
              "   'subject': 'Danielle Darrieux'},\n",
              "  'paraphrase_prompts': ['Danielle Darrieux, a native',\n",
              "   'Danielle Darrieux spoke the language'],\n",
              "  'neighborhood_prompts': ['The mother tongue of Léon Blum is',\n",
              "   'The native language of Montesquieu is',\n",
              "   'François Bayrou, a native',\n",
              "   'The native language of Raymond Barre is',\n",
              "   'Michel Rocard is a native speaker of',\n",
              "   'Jacques Chaban-Delmas is a native speaker of',\n",
              "   'The native language of François Bayrou is',\n",
              "   'Maurice Genevoix, speaker of',\n",
              "   'The mother tongue of François Bayrou is',\n",
              "   'Melchior de Vogüé, speaker of'],\n",
              "  'attribute_prompts': ['J.\\xa0R.\\xa0R. Tolkien is a native speaker of',\n",
              "   'The mother tongue of Douglas Adams is',\n",
              "   'The mother tongue of Paul McCartney is',\n",
              "   'Elvis Presley is a native speaker of',\n",
              "   'Barack Obama, speaker of',\n",
              "   'Douglas Adams, speaker of',\n",
              "   'Meryl Streep, a native',\n",
              "   'George Orwell spoke the language',\n",
              "   'George Washington, a native',\n",
              "   'Michael Jackson, a native'],\n",
              "  'generation_prompts': [\"Danielle Darrieux's mother tongue is\",\n",
              "   'Where Danielle Darrieux is from, people speak the language of',\n",
              "   \"Danielle Darrieux's mother tongue is\",\n",
              "   'Danielle Darrieux was born in',\n",
              "   \"Danielle Darrieux's mother tongue is\",\n",
              "   \"Danielle Darrieux's mother tongue is\",\n",
              "   'Danielle Darrieux was born in',\n",
              "   'Where Danielle Darrieux is from, people speak the language of',\n",
              "   'Danielle Darrieux was born in',\n",
              "   'Danielle Darrieux was born in']},\n",
              " {'case_id': 1,\n",
              "  'pararel_idx': 19501,\n",
              "  'requested_rewrite': {'prompt': 'The official religion of {} is',\n",
              "   'relation_id': 'P140',\n",
              "   'target_new': {'str': 'Islam', 'id': 'Q432'},\n",
              "   'target_true': {'str': 'Christianity', 'id': 'Q5043'},\n",
              "   'subject': 'Edwin of Northumbria'},\n",
              "  'paraphrase_prompts': ['Edwin of Northumbria follows the religion of',\n",
              "   'Edwin of Northumbria is affiliated with the religion'],\n",
              "  'neighborhood_prompts': ['The official religion of Charles Aznavour is',\n",
              "   'Nicolas Sarkozy is affiliated with the religion',\n",
              "   'Andrew Johnson is affiliated with the religion',\n",
              "   'The official religion of Paul is',\n",
              "   'Ringo Starr is follower of',\n",
              "   'The official religion of Nicolas Sarkozy is',\n",
              "   'The official religion of Andrew Johnson is',\n",
              "   'Orson Welles is affiliated with the religion',\n",
              "   'Lady Gaga is follower of',\n",
              "   'Quentin Tarantino is affiliated with the religion'],\n",
              "  'attribute_prompts': ['Dave Chappelle is affiliated with the religion',\n",
              "   'Mos Def is follower of',\n",
              "   'Hema Malini is affiliated with the religion',\n",
              "   'The official religion of Mos Def is',\n",
              "   'The official religion of Benazir Bhutto is',\n",
              "   'Benazir Bhutto is follower of',\n",
              "   'Rasul Gamzatov follows the religion of',\n",
              "   'saint follows the religion of',\n",
              "   'Averroes follows the religion of',\n",
              "   'The official religion of Rasul Gamzatov is'],\n",
              "  'generation_prompts': [\"Edwin of Northumbria's religious values strongly emphasize\",\n",
              "   'Edwin of Northumbria worships',\n",
              "   'Edwin of Northumbria regularly attends religious events at the local',\n",
              "   'Edwin of Northumbria regularly attends religious events at the local',\n",
              "   'Edwin of Northumbria regularly attends religious events at the local',\n",
              "   'Edwin of Northumbria regularly attends religious events at the local',\n",
              "   'Edwin of Northumbria regularly attends religious events at the local',\n",
              "   'Edwin of Northumbria worships',\n",
              "   'Edwin of Northumbria worships',\n",
              "   'Edwin of Northumbria regularly attends religious events at the local']},\n",
              " {'case_id': 2,\n",
              "  'pararel_idx': 13704,\n",
              "  'requested_rewrite': {'prompt': '{}, the',\n",
              "   'relation_id': 'P1303',\n",
              "   'target_new': {'str': 'piano', 'id': 'Q5994'},\n",
              "   'target_true': {'str': 'guitar', 'id': 'Q6607'},\n",
              "   'subject': 'Toko Yasuda'},\n",
              "  'paraphrase_prompts': ['Toko Yasuda, performing on the',\n",
              "   'Toko Yasuda plays the instrument'],\n",
              "  'neighborhood_prompts': ['Paul McCartney plays the instrument',\n",
              "   'John Lennon, playing the',\n",
              "   'Elvis Presley, the',\n",
              "   'Douglas Adams, playing the',\n",
              "   'John Lennon performs on the',\n",
              "   'Jimi Hendrix, playing the',\n",
              "   'Ringo Starr, playing the',\n",
              "   'Leonard Cohen plays the',\n",
              "   'Bruce Springsteen, playing the',\n",
              "   'John Lennon plays the'],\n",
              "  'attribute_prompts': ['Robert Radecke performs on the',\n",
              "   'Peter Igelhoff, the',\n",
              "   'Grete von Zieritz, performing on the',\n",
              "   'Anton Rubinstein, playing the',\n",
              "   'Hauschka, playing the',\n",
              "   'Joseph Fischhof, the',\n",
              "   'Paul Badura-Skoda plays the',\n",
              "   'Richard Fall performs on the',\n",
              "   'Justus Frantz, the',\n",
              "   'Laci Boldemann, performing on the'],\n",
              "  'generation_prompts': ['Toko Yasuda produces the most amazing music on the',\n",
              "   'Toko Yasuda is known for',\n",
              "   'Toko Yasuda is known for',\n",
              "   'Toko Yasuda is incredible at',\n",
              "   'Toko Yasuda is incredible at',\n",
              "   'Toko Yasuda produces the most amazing music on the',\n",
              "   'Toko Yasuda produces the most amazing music on the',\n",
              "   'Toko Yasuda is known for',\n",
              "   'Toko Yasuda produces the most amazing music on the',\n",
              "   'Toko Yasuda is incredible at']}]"
            ]
          },
          "execution_count": 10,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "train_data[:3]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WVq5tjUDEzyf"
      },
      "outputs": [],
      "source": [
        "MODEL_NAME = 'gpt2-xl'\n",
        "ALG_NAME= 'ROME'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "4a7b9b5d454b4549b44c46f1ba10cc1c",
            "92d6c8cf5c234326aebe08e8464d71b7",
            "f24522f15ca2485e9aa2d1aa79a3ed5c",
            "e4d070205c7e476ca1adc928f08b7f8d",
            "4d673da55feb4a9c815af1d160f85f69",
            "623c1e872a1848abb14decdcde3026c4",
            "634700a6966346ea899e3290aaf281f6",
            "9e0df70a0a284f06ac86e3cdd0486978",
            "c5f0eae0fb3d4f67a660f54b6f7114e4",
            "c3c778b0a6034413a74ef2d49f28ab10",
            "80a04a85f6e44281be83a36c31675816",
            "af213812d5fa42dc8303d1b96e4e9998",
            "1b9a741f83cc44c9842978db63b32f37",
            "e0df171ffbce4570a2ecf6f512b2c05f",
            "2cd629788e894e2887a84f73d8d52dc7",
            "47f45861e3ce4cd2907597dcbb5a266c",
            "bb12f98cda814bc7b1051a11bdb18768",
            "6c0764a34d9b4b378966d0a94237dd36",
            "686404434b894dce91c4a176daf49153",
            "ee7543804c0542cb9701b1a78a2294ff",
            "69667fe3ad84487fb7e06586c9b07155",
            "12bcde69e47b4a238a0fe82fe5843675",
            "51b866253fc04d2586e1f3b54d8d657c",
            "a78aa980c15641ce9b051230a67de2a6",
            "dd055ae85de44a2a877f50751f82f35b",
            "e68f7b2dc0714e538ca49eb658be8707",
            "ce472f84f3f743cead47e840a8b2ec99",
            "5a44253707b649b2890e041f524ba4d3",
            "faab1d63fa9645a9be541c85e2ffb821",
            "347ca81f61094f4b972473433ee3c88e",
            "8ec3dc4ea6074dec915028e2e15aefb9",
            "9254637351314f68b53060b1cc2794a8",
            "20d26e996c4847e3ad1aa925c38b70a6",
            "574390b667204afab899e4622926d093",
            "a3331077e59e41e9a0b3f88d1ebe254f",
            "0a3ed04d8ef44b56b13755d5e77972d0",
            "409d4b66256b4bb0806d49635bd7835a",
            "05c45ce63a754b7d8c315823e76e5840",
            "cef588f754714f408736c3c2928974e7",
            "fa308a3eea754e6baab346a5438afcc1",
            "f98d9832aa4b4b6e8a3a5e099c573ad9",
            "86779cf77f1a428c8af73e3c8cfb4391",
            "e48fb9305d6d4cec81957b8080e65454",
            "c35dc0ff8c384997b73696436df326f8",
            "50d3ab4ddae64a2d988ca8380d4a8197",
            "f9d5cfb5bc054091bb4d00b8a4ff1f1f",
            "5266af0a65184715997357006343db3b",
            "1dc7bc1afa7946fb9809601e20e00198",
            "30e9e28fbf3249439a62667ec73476b3",
            "f5d22ec523844edc91d948d20e43a86b",
            "afafb5b553674914b75e8f125b16dd82",
            "b64b5e28f43044a6bb0d525752f4d9c5",
            "e1b12006e21c48ce8ee7b465e59bee70",
            "5cde6f35b747447c9d4f3f17bfdbbf7c",
            "622d46bff3954107978d5193f33b4c88",
            "0e2f786de42640e9948759c044df759a",
            "072699088f4142e88a3f59bae3ca1a2f",
            "50eedfd064ed4507b8e678b01a75383c",
            "bd5f84eed49542c08a229d90a39a77ca",
            "45ac719fa2884f808d16e9ee29eac770",
            "1446963885694ffcb3a750be7bd264c5",
            "170690318d51470c9e2fd9b2c4f35628",
            "73326236852a4cfe8da64fd3a3cf6bdc",
            "03aaa1e7dc854194a78e2c1918ff740e",
            "f29c4b89a43f4537ac7731cb2e05caaa",
            "a57980e336b94ccc8be0431ce2a7da2c",
            "d8d4187bc9004e2d8d10087efc35ac2f",
            "dc16d1aa80214365916631cb2b324bdb",
            "743dba07fa7945109440a28a5324d377",
            "6a2bd8db81824ce18b81fb3580eda78d",
            "359e7096feae497986bd85c5b37ced15",
            "8f3f6454b955442fb3ab456c5f9e4375",
            "7650e761fde0480cb7aeac2b2e27b436",
            "04a121d66043492daefed58b3ebc9c75",
            "616ee66f4fc54e05a6c38f90131bfafb",
            "ce1e18d1fd464aae9ea47d68eaae5493",
            "e6bd3b3dc83d450bacba1228976f1a0b"
          ]
        },
        "id": "WvNcznvKE-87",
        "outputId": "7921b939-6190-4a16-bc49-a715b0079ef6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/689 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "4a7b9b5d454b4549b44c46f1ba10cc1c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model.safetensors:   0%|          | 0.00/6.43G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "af213812d5fa42dc8303d1b96e4e9998"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "generation_config.json:   0%|          | 0.00/124 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "51b866253fc04d2586e1f3b54d8d657c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/26.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "574390b667204afab899e4622926d093"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "vocab.json:   0%|          | 0.00/1.04M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "50d3ab4ddae64a2d988ca8380d4a8197"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0e2f786de42640e9948759c044df759a"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer.json:   0%|          | 0.00/1.36M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d8d4187bc9004e2d8d10087efc35ac2f"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "GPT2Config {\n",
              "  \"_name_or_path\": \"gpt2-xl\",\n",
              "  \"activation_function\": \"gelu_new\",\n",
              "  \"architectures\": [\n",
              "    \"GPT2LMHeadModel\"\n",
              "  ],\n",
              "  \"attn_pdrop\": 0.1,\n",
              "  \"bos_token_id\": 50256,\n",
              "  \"embd_pdrop\": 0.1,\n",
              "  \"eos_token_id\": 50256,\n",
              "  \"initializer_range\": 0.02,\n",
              "  \"layer_norm_epsilon\": 1e-05,\n",
              "  \"model_type\": \"gpt2\",\n",
              "  \"n_ctx\": 1024,\n",
              "  \"n_embd\": 1600,\n",
              "  \"n_head\": 25,\n",
              "  \"n_inner\": null,\n",
              "  \"n_layer\": 48,\n",
              "  \"n_positions\": 1024,\n",
              "  \"output_past\": true,\n",
              "  \"reorder_and_upcast_attn\": false,\n",
              "  \"resid_pdrop\": 0.1,\n",
              "  \"scale_attn_by_inverse_layer_idx\": false,\n",
              "  \"scale_attn_weights\": true,\n",
              "  \"summary_activation\": null,\n",
              "  \"summary_first_dropout\": 0.1,\n",
              "  \"summary_proj_to_labels\": true,\n",
              "  \"summary_type\": \"cls_index\",\n",
              "  \"summary_use_proj\": true,\n",
              "  \"task_specific_params\": {\n",
              "    \"text-generation\": {\n",
              "      \"do_sample\": true,\n",
              "      \"max_length\": 50\n",
              "    }\n",
              "  },\n",
              "  \"transformers_version\": \"4.38.2\",\n",
              "  \"use_cache\": true,\n",
              "  \"vocab_size\": 50257\n",
              "}"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ],
      "source": [
        "model, tok = (\n",
        "    AutoModelForCausalLM.from_pretrained(MODEL_NAME, low_cpu_mem_usage=IS_COLAB).to(\n",
        "        \"cuda\"\n",
        "    ),\n",
        "    AutoTokenizer.from_pretrained(MODEL_NAME),\n",
        ")\n",
        "tok.pad_token = tok.eos_token\n",
        "model.config"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1Obwg8UuFGal",
        "outputId": "e7659c43-3285-4710-b40f-ba5af82ea4fb"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "No model weights to restore: name 'orig_weights' is not defined\n"
          ]
        }
      ],
      "source": [
        "try:\n",
        "    with torch.no_grad():\n",
        "        for k, v in orig_weights.items():\n",
        "            nethook.get_parameter(model, k)[...] = v\n",
        "    print(\"Original model restored\")\n",
        "except NameError as e:\n",
        "    print(f\"No model weights to restore: {e}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "udZDzvMVFHZs"
      },
      "outputs": [],
      "source": [
        "from rome import ROMEHyperParams, apply_rome_to_model\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sWoqfL4PI9TG",
        "outputId": "8722cf61-302b-4c06-a1d6-012229b9e72c"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "ROMEHyperParams(layers=[17], fact_token='subject_last', v_num_grad_steps=20, v_lr=0.5, v_loss_layer=47, v_weight_decay=0.5, clamp_norm_factor=4, kl_factor=0.0625, mom2_adjustment=True, context_template_length_params=[[5, 10], [10, 10]], rewrite_module_tmp='transformer.h.{}.mlp.c_proj', layer_module_tmp='transformer.h.{}', mlp_module_tmp='transformer.h.{}.mlp', attn_module_tmp='transformer.h.{}.attn', ln_f_module='transformer.ln_f', lm_head_module='transformer.wte', mom2_dataset='wikipedia', mom2_n_samples=100000, mom2_dtype='float32')"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ],
      "source": [
        "def load_alg(alg_name):\n",
        "  if alg_name == \"ROME\":\n",
        "        return ROMEHyperParams, apply_rome_to_model, \"ROME\", \"\"\n",
        "\n",
        "RewritingParamsClass, apply_method, hparams_prefix, hparams_suffix = load_alg(ALG_NAME)\n",
        "params_name= f'hparams/{hparams_prefix}/{MODEL_NAME}.json'\n",
        "hparams = RewritingParamsClass.from_json(params_name)\n",
        "hparams"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7Al-e1DMN-jA"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import scipy\n",
        "import nltk\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from dsets import AttributeSnippets\n",
        "from util.generate import generate_fast"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3ipr4e1mfuQr",
        "outputId": "2b3f5da2-ff39-461e-c540-3b012eafef35"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ],
      "source": [
        "import nltk\n",
        "nltk.download('punkt')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wei3AjGKZKUk"
      },
      "outputs": [],
      "source": [
        "from itertools import chain\n",
        "\n",
        "import nltk\n",
        "import numpy as np\n",
        "import scipy\n",
        "import torch\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
        "\n",
        "from dsets import AttributeSnippets\n",
        "from util.generate import generate_fast\n",
        "from util.perplexity import perplexity"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wENrC1TvLrgJ",
        "outputId": "9dce6350-bf8e-442e-a239-c584246f2b70"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2024-04-16 14:58:11--  https://raw.githubusercontent.com/princeton-nlp/MQuAKE/main/datasets/MQuAKE-CF-3k.json\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 16055945 (15M) [text/plain]\n",
            "Saving to: ‘MQuAKE-CF-3k.json’\n",
            "\n",
            "MQuAKE-CF-3k.json   100%[===================>]  15.31M  --.-KB/s    in 0.09s   \n",
            "\n",
            "2024-04-16 14:58:12 (175 MB/s) - ‘MQuAKE-CF-3k.json’ saved [16055945/16055945]\n",
            "\n"
          ]
        }
      ],
      "source": [
        "!wget \"https://raw.githubusercontent.com/princeton-nlp/MQuAKE/main/datasets/MQuAKE-CF-3k.json\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JQRqb1LQL-eM"
      },
      "outputs": [],
      "source": [
        "import json\n",
        "with open('/content/rome/MQuAKE-CF-3k.json', 'r') as f:\n",
        "    dataset = json.load(f)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RAMY32cvMM52",
        "outputId": "c17e1d12-00b5-4df7-dd24-8f9dc3c79515"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'case_id': 2001,\n",
              " 'requested_rewrite': [{'prompt': '{} plays the position of',\n",
              "   'relation_id': 'P413',\n",
              "   'target_new': {'str': 'goalkeeper', 'id': 'Q201330'},\n",
              "   'target_true': {'str': 'midfielder', 'id': 'Q193592'},\n",
              "   'subject': 'Arturo Vidal',\n",
              "   'question': 'What position does Arturo Vidal play?'},\n",
              "  {'prompt': '{} is associated with the sport of',\n",
              "   'relation_id': 'P641',\n",
              "   'target_new': {'str': 'baseball', 'id': 'Q5369'},\n",
              "   'target_true': {'str': 'association football', 'id': 'Q2736'},\n",
              "   'subject': 'goalkeeper',\n",
              "   'question': 'Which sport is goalkeeper associated with?'},\n",
              "  {'prompt': '{} was created in the country of',\n",
              "   'relation_id': 'P495',\n",
              "   'target_new': {'str': 'United Kingdom', 'id': 'Q145'},\n",
              "   'target_true': {'str': 'United States of America', 'id': 'Q30'},\n",
              "   'subject': 'baseball',\n",
              "   'question': 'Which country was baseball created in?'},\n",
              "  {'prompt': '{} is located in the continent of',\n",
              "   'relation_id': 'P30',\n",
              "   'target_new': {'str': 'Asia', 'id': 'Q48'},\n",
              "   'target_true': {'str': 'Europe', 'id': 'Q46'},\n",
              "   'subject': 'United Kingdom',\n",
              "   'question': 'Which continent is United Kingdom located in?'}],\n",
              " 'questions': [\"What continent is the country of origin of Arturo Vidal's sport located in?\",\n",
              "  \"In which continent is the country that Arturo Vidal's sport hails from?\",\n",
              "  'From which continent does the sport, which Arturo Vidal plays, originate?'],\n",
              " 'answer': 'Europe',\n",
              " 'answer_alias': ['European continent', 'Old Continent'],\n",
              " 'new_answer': 'Asia',\n",
              " 'new_answer_alias': ['Asian continent'],\n",
              " 'single_hops': [{'question': 'What position does Arturo Vidal play?',\n",
              "   'cloze': 'Arturo Vidal plays the position of',\n",
              "   'answer': 'midfielder',\n",
              "   'answer_alias': ['M', 'MF', 'mid-fielder']},\n",
              "  {'question': 'Which sport is midfielder associated with?',\n",
              "   'cloze': 'midfielder is associated with the sport of',\n",
              "   'answer': 'association football',\n",
              "   'answer_alias': ['football',\n",
              "    'association soccer',\n",
              "    'footie',\n",
              "    'footy',\n",
              "    'futbol',\n",
              "    'futebol',\n",
              "    'soccer']},\n",
              "  {'question': 'Which country was association football created in?',\n",
              "   'cloze': 'association football was created in the country of',\n",
              "   'answer': 'England',\n",
              "   'answer_alias': ['ENG']},\n",
              "  {'question': 'Which continent is England located in?',\n",
              "   'cloze': 'England is located in the continent of',\n",
              "   'answer': 'Europe',\n",
              "   'answer_alias': ['European continent', 'Old Continent']}],\n",
              " 'new_single_hops': [{'question': 'What position does Arturo Vidal play?',\n",
              "   'cloze': 'Arturo Vidal plays the position of',\n",
              "   'answer': 'goalkeeper',\n",
              "   'answer_alias': ['football goalkeeper',\n",
              "    'GK',\n",
              "    'goalie',\n",
              "    'soccer goalkeeper']},\n",
              "  {'question': 'Which sport is goalkeeper associated with?',\n",
              "   'cloze': 'goalkeeper is associated with the sport of',\n",
              "   'answer': 'baseball',\n",
              "   'answer_alias': ['⚾', \"America's pastime\"]},\n",
              "  {'question': 'Which country was baseball created in?',\n",
              "   'cloze': 'baseball was created in the country of',\n",
              "   'answer': 'United Kingdom',\n",
              "   'answer_alias': ['Britain',\n",
              "    'UK',\n",
              "    'G. B.',\n",
              "    'GBR',\n",
              "    'Great Britain',\n",
              "    'GB',\n",
              "    'G.B.',\n",
              "    'United Kingdom of Great Britain and Northern Ireland',\n",
              "    'G B',\n",
              "    'G B R',\n",
              "    'G.B.R.',\n",
              "    '🇬🇧',\n",
              "    'Great Britain and Northern Ireland',\n",
              "    'The UK',\n",
              "    'The United Kingdom of Great Britain and Northern Ireland',\n",
              "    'U K',\n",
              "    'U. K.',\n",
              "    'U.K.']},\n",
              "  {'question': 'Which continent is United Kingdom located in?',\n",
              "   'cloze': 'United Kingdom is located in the continent of',\n",
              "   'answer': 'Asia',\n",
              "   'answer_alias': ['Asian continent']}],\n",
              " 'orig': {'triples': [['Q192671', 'P413', 'Q193592'],\n",
              "   ['Q193592', 'P641', 'Q2736'],\n",
              "   ['Q2736', 'P495', 'Q21'],\n",
              "   ['Q21', 'P30', 'Q46']],\n",
              "  'triples_labeled': [['Arturo Vidal',\n",
              "    'position played on team / speciality',\n",
              "    'midfielder'],\n",
              "   ['midfielder', 'sport', 'association football'],\n",
              "   ['association football', 'country of origin', 'England'],\n",
              "   ['England', 'continent', 'Europe']],\n",
              "  'new_triples': [['Q192671', 'P413', 'Q201330'],\n",
              "   ['Q201330', 'P641', 'Q5369'],\n",
              "   ['Q5369', 'P495', 'Q145'],\n",
              "   ['Q145', 'P30', 'Q48']],\n",
              "  'new_triples_labeled': [['Arturo Vidal',\n",
              "    'position played on team / speciality',\n",
              "    'goalkeeper'],\n",
              "   ['goalkeeper', 'sport', 'baseball'],\n",
              "   ['baseball', 'country of origin', 'United Kingdom'],\n",
              "   ['United Kingdom', 'continent', 'Asia']],\n",
              "  'edit_triples': [['Q192671', 'P413', 'Q201330'],\n",
              "   ['Q201330', 'P641', 'Q5369'],\n",
              "   ['Q5369', 'P495', 'Q145'],\n",
              "   ['Q145', 'P30', 'Q48']]}}"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ],
      "source": [
        "dataset[2000]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mKJccbYkuWt8",
        "outputId": "bba40987-a4f4-445f-8b70-310a83e3b658"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[{'prompt': '{} plays the position of', 'subject': 'Arturo Vidal', 'target_new': {'str': 'goalkeeper'}}, {'prompt': '{} was founded by', 'subject': 'Ford Germany', 'target_new': {'str': 'Aristotle'}}]\n",
            "[[\"What continent is the country of origin of Arturo Vidal's sport located in? Answer:\", \"In which continent is the country that Arturo Vidal's sport hails from? Answer:\", 'From which continent does the sport, which Arturo Vidal plays, originate? Answer:'], ['What is the capital of the country where the child company of the company founded by Ford Germany holds citizenship? Answer:', 'Which city is the capital of the country where the subsidiary of the company founded by Ford Germany is a citizen? Answer:', 'In which capital city is the subsidiary of the company founded by Ford Germany located? Answer:'], [\"What is the official language of the country where the creator of Derek Shepherd's spouse holds citizenship? Answer:\", \"In which country, whose official language is x4, is the creator of Derek Shepherd's spouse a citizen of? Answer:\", \"Which language is the official means of communication in the country of citizenship of the creator of Derek Shepherd's spouse? Answer:\"], ['What is the capital of the country where the sport played by Andre Miller originated? Answer:', 'Which city serves as the capital of the country where the sport played by Andre Miller originated? Answer:', 'What city is the capital of the country where the sport played by Andre Miller was born? Answer:'], [\"What is the capital of the country that Rony Seikaly's sport originates from? Answer:\", \"In which city can you find the capital of the country that Rony Seikaly's sport originates from? Answer:\", \"Which city serves as the capital of the country that Rony Seikaly's sport comes from? Answer:\"], ['What city is the capital of the country to which the spouse of the author of Little Dorrit belongs? Answer:', 'In which city is the capital of the country where the spouse of the author of Little Dorrit is a citizen? Answer:', 'Which city is the capital of the country where the partner of the author of Little Dorrit holds citizenship? Answer:'], ['What is the capital of the country where the sport that the head coach of Miami Heat coaches in was originated? Answer:', 'Which city serves as the capital of the country where the sport of the head coach of Miami Heat originated? Answer:', 'In which capital city did the sport coached by the head coach of Miami Heat originate? Answer:'], [\"In which continent is the country of origin of Marten Stekelenburg's sport located? Answer:\", \"From which continent does the country of origin of Marten Stekelenburg's specialization belong to? Answer:\", \"Which continent does the sport's country of origin, in which Marten Stekelenburg's specialization belongs to, lie? Answer:\"], ['Which city serves as the capital of the country that is the birthplace of the sport played by Kirk Hinrich? Answer:', 'What is the capital of the country where the sport played by Kirk Hinrich originated? Answer:', 'In which city is the capital of the country where the sport played by Kirk Hinrich was developed? Answer:'], [\"What is the capital of the country that Fred McGriff's sport originated from? Answer:\", 'Which city serves as the capital of the country where the sport of Fred McGriff was first played? Answer:', 'Where did the sport that Fred McGriff specializes in originate, and what is the capital of that country? Answer:'], [\"In which continent was the CEO of GW-BASIC's developer born? Answer:\", 'What continent is the birthplace of the chief executive officer of the developer of GW-BASIC located in? Answer:', 'From which continent does the CEO of the developer of GW-BASIC originate? Answer:'], ['What continent is the country of citizenship of the director/manager of the performer of \"Back in the U.S.S.R.\" located in? Answer:', 'In which continent was the director/manager of the performer of \"Back in the U.S.S.R.\" born? Answer:', 'What is the continent of origin of the director/manager of the performer of \"Back in the U.S.S.R.\"? Answer:'], ['What is the capital of the country where the director/manager of \"Jimmy Kimmel Live!\" holds citizenship? Answer:', 'In which city is the head of \"Jimmy Kimmel Live!\"\\'s original broadcaster\\'s country of citizenship located? Answer:', 'What is the capital city of the country where the director/manager of \"Jimmy Kimmel Live!\" comes from? Answer:'], ['Which city is the capital of the country whose citizen founded the manufacturer of Ford Probe? Answer:', \"In what city was the Ford Probe car maker's founder's country of citizenship's capital located? Answer:\", \"What city serves as the capital of the country where the founder of Ford Probe's manufacturer was a citizen? Answer:\"], ['What is the capital of the country of citizenship of the person who founded the manufacturer of the Ford Laser? Answer:', 'Which city serves as the capital of the country whose citizen founded the manufacturer of the Ford Laser? Answer:', 'In what city is the headquarters of the country of citizenship of the founder of the manufacturer of the Ford Laser located? Answer:'], ['What is the official language of the country of citizenship of the child of the founder of Clinton Foundation? Answer:', 'Which language serves as the official language in the country whose citizen is the child of the founder of Clinton Foundation? Answer:', 'What language is the official one in the country where the child of the founder of Clinton Foundation holds citizenship? Answer:'], ['What city serves as the capital of the country where the CEO of MobileMe holds citizenship? Answer:', 'Which city is the capital of the country of citizenship of the CEO of MobileMe? Answer:', 'What is the capital of the country which the CEO of MobileMe is a citizen of? Answer:'], ['What is the capital city of the country from which the founder of the International Judo Federation hails? Answer:', 'Which city is the capital of the country where the founder of the International Judo Federation holds citizenship? Answer:', 'From which city is the founder of the International Judo Federation who has the citizenship of a country whose capital is x4? Answer:'], ['What is the capital of the country where the CEO of the developer of Apple Music holds the citizenship? Answer:', 'In which city is the CEO of the developer of Apple Music, who is a citizen of which country, situated? Answer:', 'What is the capital city of the country where the CEO of the developer of Apple Music holds the citizenship? Answer:'], [\"Which continent is the birthplace of the CEO of Xbox's manufacturer located in? Answer:\", 'From which continent does the CEO of the manufacturer of Xbox come from? Answer:', 'In which continent was the CEO of the manufacturer of Xbox born? Answer:'], [\"What is the capital city of the country of origin of Yossi Benayoun's sport? Answer:\", \"In which city is the capital of the country where Yossi Benayoun's sport originated? Answer:\", 'What is the capital of the country where the sport played by Yossi Benayoun originated? Answer:'], ['What is the capital city of the country where the sport played by Víctor Valdés originates from? Answer:', 'Which city serves as the capital of the country where the sport played by Víctor Valdés originates from? Answer:', 'In which city is the capital of the country where the sport played by Víctor Valdés originates from located? Answer:'], ['What is the capital of the country where the director or manager of the performer of \"Yesterday\" holds citizenship? Answer:', 'The director or manager of the performer of \"Yesterday\" is a citizen of which country\\'s capital? Answer:', 'In what city is the capital of the country of citizenship of the director or manager of the performer of \"Yesterday\" located? Answer:'], ['What capital city is the country of citizenship of the director/manager of \"Galavant\"? Answer:', 'The director/manager of \"Galavant\" holds citizenship in which country whose capital is what city? Answer:', 'What is the name of the capital city of the country of citizenship of the director/manager of \"Galavant\"? Answer:'], [\"What is the capital city of the country where Jens Lehmann's sport originated? Answer:\", 'Which city acts as the capital of the country where the sport of Jens Lehmann was first played? Answer:', \"In which city is the capital of the country where Jens Lehmann's sport came from? Answer:\"], ['In which continent was the birthplace of the chief executive officer who developed Windows 8? Answer:', 'Which continent is the CEO, who is responsible for Windows 8, originally from? Answer:', 'The place of birth of the chief executive officer responsible for Windows 8 is located in what continent? Answer:'], ['What is the official language of the country of citizenship of the spouse of the author of the Star Trek canon? Answer:', 'What language is spoken officially in the native country of citizenship of the spouse of the person who authored the Star Trek canon? Answer:', 'Which language is the official language of the country of nationality of the partner of the author of Star Trek canon? Answer:'], ['In which continent was the creator of the organization that founded the Death Eater group a citizen of? Answer:', 'From which continent did the creator of the organization that created the Death Eaters belong to? Answer:', 'What continent was the country that the creator of the organization who founded the Death Eater was a citizen of located in? Answer:'], ['What is the official language of the religion that is associated with the spouse of the performer of \"Diamond Dogs\"? Answer:', 'Which language is considered to be the official language of the religion that is connected to the partner of the performer of \"Diamond Dogs\"? Answer:', 'What language is used as the official language of the religion followed by the spouse of the performer of \"Diamond Dogs\"? Answer:'], ['What is the capital of the country of origin of the sport played by Jon Garland? Answer:', 'Which city serves as the capital of the country where the sport played by Jon Garland originated? Answer:', \"What city is the capital of the country from which the sport of Jon Garland's specialty originated? Answer:\"], ['What is the official language of the country where the director/manager of \"Ticket to Ride\" holds citizenship? Answer:', 'Which language is officially spoken in the country of citizenship of the director/manager of \"Ticket to Ride\"? Answer:', 'What is the official tongue of the country to which the director/manager of \"Ticket to Ride\" belongs? Answer:'], [\"What is the capital of the country whose citizen is the chief executive officer of GarageBand's developer? Answer:\", \"Which city serves as the capital of the country whose citizen is the CEO of GarageBand's developer? Answer:\", 'In which city is the capital of the country whose citizen is the CEO of the developer of GarageBand? Answer:'], ['What is the capital of the country that the sport played by Brad Miller originates from? Answer:', 'From which country does the sport played by Brad Miller come and what is its capital? Answer:', 'Which city serves as the capital of the country from which the sport played by Brad Miller originates? Answer:'], ['What is the capital city of the country where the director/manager of \"Loving\" holds citizenship? Answer:', 'In which city is the capital of the country where the director/manager of \"Loving\" comes from? Answer:', 'Which city is the capital of the country that the director/manager of \"Loving\" is a citizen of? Answer:'], [\"What continent is the birthplace of Active Directory's CEO located in? Answer:\", 'Which continent is the CEO of Active Directory from? Answer:', \"In which continent was Active Directory's CEO born? Answer:\"], ['What continent is the country of citizenship of the spouse of the author of \"The Selfish Gene\" located on? Answer:', 'Which continent houses the citizen country of the spouse of the author of \"The Selfish Gene\"? Answer:', 'On what continent is the country of citizenship of the spouse of \"The Selfish Gene\" author located? Answer:'], ['What capital city is the country of citizenship of the director or manager of the performers of \"With the Beatles\"? Answer:', 'What is the name of the capital of the country where the person who directed or managed the performers of \"With the Beatles\" is a citizen? Answer:', 'In which capital is the country that the director or manager of \"With the Beatles\" performers is a citizen? Answer:'], ['What is the capital city of the country where the sport played by Nate Robinson originated? Answer:', \"Which city is the capital of the country where the sport of Nate Robinson's specialization was invented? Answer:\", 'In which city is the capital of the country where the sport played by Nate Robinson was invented? Answer:'], ['Which continent is the birthplace of the chief executive officer of the developer of the File Explorer? Answer:', 'From which continent does the chief executive officer of the developer of the File Explorer come? Answer:', 'In which continent was the chief executive officer of the developer of the File Explorer born? Answer:'], ['What is the capital city of the home country of the sport played by Sepp Maier? Answer:', \"In which city is the capital of the country from which the sport of Sepp Maier's position on a team originates? Answer:\", 'Where is the capital of the country where the sport for which Sepp Maier played a certain position on a team was invented? Answer:'], ['What is the capital of the country of citizenship of the director/manager associated with \"Revolution 9\" performers? Answer:', 'What city serves as the capital of the home country of the director/manager of \"Revolution 9\" performers? Answer:', 'Which city is the capital of the country where the director/manager who worked with the performers of \"Revolution 9\" comes from? Answer:'], ['What is the capital of the country where the sport played by Panagiotis Fasoulas originated? Answer:', 'Which city is the capital of the country where the sport of Panagiotis Fasoulas originated? Answer:', 'In which capital city was the sport that Panagiotis Fasoulas played originally developed? Answer:'], ['What is the official language of the country where the director/manager of \"Help!\" holds a citizenship? Answer:', 'In which language do people from the country of citizenship of the director/manager of \"Help!\" communicate officially? Answer:', 'What language is officially used in the country where the director/manager of \"Help!\" is a citizen? Answer:'], ['What is the capital city of the country of origin of the sport played by Kendrys Morales? Answer:', \"Kendrys Morales represents a sport that originated in which country's capital? Answer:\", 'In which city is the headquarters of the country where the sport played by Kendrys Morales originates located? Answer:'], [\"What is the capital of the country that the sport coached by Galatasaray S.K.'s head coach originates from? Answer:\", \"In which city is the capital of the country from which Galatasaray S.K.'s head coach's sport originates? Answer:\", 'What city serves as the capital of the country where the sport coached by the head coach of Galatasaray S.K. was founded? Answer:'], [\"What is the capital of the country where the sport of Shaun Marcum's specialty originates from? Answer:\", \"Which city is the capital of the country where Shaun Marcum's sport of choice was first played? Answer:\", 'What is the name of the capital of the country where the sport Shaun Marcum specialized in was created? Answer:'], [\"What is the official language of the country to which the creator of Eddard Stark's spouse belongs? Answer:\", \"In which language are official documents written in the country of citizenship of the creator of Eddard Stark's spouse? Answer:\", \"What is the language of the government of the country to which the creator of Eddard Stark's spouse belongs? Answer:\"], ['In which continent was the chairperson who developed XML Schema a citizen of? Answer:', \"What is the continent of origin of the developer of XML Schema's chairperson? Answer:\", 'From which continent does the country of citizenship of the chairperson who developed XML Schema belong to? Answer:'], ['In which continent was the sport originated of the current head coach of the Football Association of Ireland? Answer:', 'Which continent does the country of origin of the current head coach of the Football Association of Ireland belong to? Answer:', \"From which continent does the current head coach of the Football Association of Ireland's country of origin come? Answer:\"], ['What continent is the country of citizenship of the person who directed/managed the performance of \"Get Back\" located in? Answer:', 'In which geographical region is the country of citizenship of the director/manager of \"Get Back\"? Answer:', 'Where is the country of citizenship of the director/manager of the performance that included \"Get Back\" situated within the world? Answer:']]\n",
            "['What position does Arturo Vidal play? Answer:', 'Who founded Ford Germany? Answer:', 'Who is Derek Shepherd married to? Answer:', 'What position does Andre Miller play? Answer:', 'What position does Rony Seikaly play? Answer:', 'Who is the author of Little Dorrit? Answer:', 'Who is the head coach of Miami Heat? Answer:', 'What position does Marten Stekelenburg play? Answer:', 'What position does Kirk Hinrich play? Answer:', 'What position does Fred McGriff play? Answer:', 'Who is the developer of GW-BASIC? Answer:', 'Who performed Back in the U.S.S.R.? Answer:', 'Who is the original broadcaster of Jimmy Kimmel Live!? Answer:', 'Which company is Ford Probe produced by? Answer:', 'Which company is Ford Laser produced by? Answer:', 'Who founded Clinton Foundation? Answer:', 'Who is the developer of MobileMe? Answer:', 'Which sport is International Judo Federation associated with? Answer:', 'Who is the developer of Apple Music? Answer:', 'Which company is Xbox produced by? Answer:', 'What position does Yossi Benayoun play? Answer:', 'What position does Víctor Valdés play? Answer:', 'Who performed Yesterday? Answer:', 'Who is the original broadcaster of Galavant? Answer:', 'What position does Jens Lehmann play? Answer:', 'Who is the developer of Windows 8? Answer:', 'Who is the author of Star Trek canon? Answer:', 'Who founded Death Eater? Answer:', 'Who performed Diamond Dogs? Answer:', 'What position does Jon Garland play? Answer:', 'Who performed Ticket to Ride? Answer:', 'Who is the developer of GarageBand? Answer:', 'What position does Brad Miller play? Answer:', 'Who is the original broadcaster of Loving? Answer:', 'Who is the developer of Active Directory? Answer:', 'Who is the author of The Selfish Gene? Answer:', 'Who performed With the Beatles? Answer:', 'What position does Nate Robinson play? Answer:', 'Who is the developer of File Explorer? Answer:', 'What position does Sepp Maier play? Answer:', 'Who performed Revolution 9? Answer:', 'What position does Panagiotis Fasoulas play? Answer:', 'Who performed Help!? Answer:', 'What position does Kendrys Morales play? Answer:', 'Who is the head coach of Galatasaray S.K.? Answer:', 'What position does Shaun Marcum play? Answer:', 'Who is Eddard Stark married to? Answer:', 'Who is the developer of XML Schema? Answer:', 'Who is the head coach of Football Association of Ireland? Answer:', 'Who performed Get Back? Answer:']\n",
            "[['Europe', 'European continent', 'Old Continent'], ['Washington, D.C.', 'Washington', 'The District', 'D.C.', 'DC', 'District of Columbia', 'Washington D.C.', 'Washington, DC', 'US-DC', 'Washington DC', 'Washington, D.C., U.S.', 'Washington, District of Columbia'], ['American English', 'AmE', 'en_us', 'en_US', 'en-us', 'en-US', 'English (United States)', 'U.S. English', 'United States English', 'US English'], ['Washington, D.C.', 'Washington', 'The District', 'D.C.', 'DC', 'District of Columbia', 'Washington D.C.', 'Washington, DC', 'US-DC', 'Washington DC', 'Washington, D.C., U.S.', 'Washington, District of Columbia'], ['Washington, D.C.', 'Washington', 'The District', 'D.C.', 'DC', 'District of Columbia', 'Washington D.C.', 'Washington, DC', 'US-DC', 'Washington DC', 'Washington, D.C., U.S.', 'Washington, District of Columbia'], ['London', 'London, United Kingdom', 'Greater London', 'London, England', 'London U.K.', 'London UK', 'London, UK'], ['Washington, D.C.', 'Washington', 'The District', 'D.C.', 'DC', 'District of Columbia', 'Washington D.C.', 'Washington, DC', 'US-DC', 'Washington DC', 'Washington, D.C., U.S.', 'Washington, District of Columbia'], ['Europe', 'European continent', 'Old Continent'], ['Washington, D.C.', 'Washington', 'The District', 'D.C.', 'DC', 'District of Columbia', 'Washington D.C.', 'Washington, DC', 'US-DC', 'Washington DC', 'Washington, D.C., U.S.', 'Washington, District of Columbia'], ['Washington, D.C.', 'Washington', 'The District', 'D.C.', 'DC', 'District of Columbia', 'Washington D.C.', 'Washington, DC', 'US-DC', 'Washington DC', 'Washington, D.C., U.S.', 'Washington, District of Columbia'], ['Asia', 'Asian continent'], ['Europe', 'European continent', 'Old Continent'], ['Washington, D.C.', 'Washington', 'The District', 'D.C.', 'DC', 'District of Columbia', 'Washington D.C.', 'Washington, DC', 'US-DC', 'Washington DC', 'Washington, D.C., U.S.', 'Washington, District of Columbia'], ['Washington, D.C.', 'Washington', 'The District', 'D.C.', 'DC', 'District of Columbia', 'Washington D.C.', 'Washington, DC', 'US-DC', 'Washington DC', 'Washington, D.C., U.S.', 'Washington, District of Columbia'], ['Washington, D.C.', 'Washington', 'The District', 'D.C.', 'DC', 'District of Columbia', 'Washington D.C.', 'Washington, DC', 'US-DC', 'Washington DC', 'Washington, D.C., U.S.', 'Washington, District of Columbia'], ['American English', 'AmE', 'en_us', 'en_US', 'en-us', 'en-US', 'English (United States)', 'U.S. English', 'United States English', 'US English'], ['Washington, D.C.', 'Washington', 'The District', 'D.C.', 'DC', 'District of Columbia', 'Washington D.C.', 'Washington, DC', 'US-DC', 'Washington DC', 'Washington, D.C., U.S.', 'Washington, District of Columbia'], ['Tokyo', 'Tokio', 'Tokyo, Japan', 'Tokei', 'Tokyo Prefecture', 'Tôkyô', 'Tōkyō', 'Tokyo (Japan)', 'Tokyo Metropolis', 'Tokyo Metropolitan prefecture', 'Tokyo-to', 'Tôkyô-to', 'Tōkyō-to'], ['Washington, D.C.', 'Washington', 'The District', 'D.C.', 'DC', 'District of Columbia', 'Washington D.C.', 'Washington, DC', 'US-DC', 'Washington DC', 'Washington, D.C., U.S.', 'Washington, District of Columbia'], ['Asia', 'Asian continent'], ['London', 'London, United Kingdom', 'Greater London', 'London, England', 'London U.K.', 'London UK', 'London, UK'], ['London', 'London, United Kingdom', 'Greater London', 'London, England', 'London U.K.', 'London UK', 'London, UK'], ['London', 'London, United Kingdom', 'Greater London', 'London, England', 'London U.K.', 'London UK', 'London, UK'], ['Washington, D.C.', 'Washington', 'The District', 'D.C.', 'DC', 'District of Columbia', 'Washington D.C.', 'Washington, DC', 'US-DC', 'Washington DC', 'Washington, D.C., U.S.', 'Washington, District of Columbia'], ['London', 'London, United Kingdom', 'Greater London', 'London, England', 'London U.K.', 'London UK', 'London, UK'], ['Asia', 'Asian continent'], ['American English', 'AmE', 'en_us', 'en_US', 'en-us', 'en-US', 'English (United States)', 'U.S. English', 'United States English', 'US English'], ['Europe', 'European continent', 'Old Continent'], ['Arabic', 'ar', 'Arabian language', 'Arabic language'], ['Washington, D.C.', 'Washington', 'The District', 'D.C.', 'DC', 'District of Columbia', 'Washington D.C.', 'Washington, DC', 'US-DC', 'Washington DC', 'Washington, D.C., U.S.', 'Washington, District of Columbia'], ['English', 'en', 'eng', 'English language'], ['Washington, D.C.', 'Washington', 'The District', 'D.C.', 'DC', 'District of Columbia', 'Washington D.C.', 'Washington, DC', 'US-DC', 'Washington DC', 'Washington, D.C., U.S.', 'Washington, District of Columbia'], ['Washington, D.C.', 'Washington', 'The District', 'D.C.', 'DC', 'District of Columbia', 'Washington D.C.', 'Washington, DC', 'US-DC', 'Washington DC', 'Washington, D.C., U.S.', 'Washington, District of Columbia'], ['Washington, D.C.', 'Washington', 'The District', 'D.C.', 'DC', 'District of Columbia', 'Washington D.C.', 'Washington, DC', 'US-DC', 'Washington DC', 'Washington, D.C., U.S.', 'Washington, District of Columbia'], ['Asia', 'Asian continent'], ['Europe', 'European continent', 'Old Continent'], ['London', 'London, United Kingdom', 'Greater London', 'London, England', 'London U.K.', 'London UK', 'London, UK'], ['Washington, D.C.', 'Washington', 'The District', 'D.C.', 'DC', 'District of Columbia', 'Washington D.C.', 'Washington, DC', 'US-DC', 'Washington DC', 'Washington, D.C., U.S.', 'Washington, District of Columbia'], ['Asia', 'Asian continent'], ['London', 'London, United Kingdom', 'Greater London', 'London, England', 'London U.K.', 'London UK', 'London, UK'], ['London', 'London, United Kingdom', 'Greater London', 'London, England', 'London U.K.', 'London UK', 'London, UK'], ['Washington, D.C.', 'Washington', 'The District', 'D.C.', 'DC', 'District of Columbia', 'Washington D.C.', 'Washington, DC', 'US-DC', 'Washington DC', 'Washington, D.C., U.S.', 'Washington, District of Columbia'], ['English', 'en', 'eng', 'English language'], ['Washington, D.C.', 'Washington', 'The District', 'D.C.', 'DC', 'District of Columbia', 'Washington D.C.', 'Washington, DC', 'US-DC', 'Washington DC', 'Washington, D.C., U.S.', 'Washington, District of Columbia'], ['London', 'London, United Kingdom', 'Greater London', 'London, England', 'London U.K.', 'London UK', 'London, UK'], ['Washington, D.C.', 'Washington', 'The District', 'D.C.', 'DC', 'District of Columbia', 'Washington D.C.', 'Washington, DC', 'US-DC', 'Washington DC', 'Washington, D.C., U.S.', 'Washington, District of Columbia'], ['American English', 'AmE', 'en_us', 'en_US', 'en-us', 'en-US', 'English (United States)', 'U.S. English', 'United States English', 'US English'], ['Europe', 'European continent', 'Old Continent'], ['Europe', 'European continent', 'Old Continent'], ['Europe', 'European continent', 'Old Continent']]\n",
            "[['baseball', '⚾', \"America's pastime\", 'midfielder', 'association football', 'United States of America', 'Europe'], ['Htin Kyaw', 'Henry Ford', 'Nicomachus', 'Myanmar', 'Montevideo'], ['Frank Herbert', 'Frank Patrick Herbert', 'Franklin Herbert', 'Franklin Patrick Herbert', 'Franklin Patrick Herbert, Jr.', 'Jr. Frank Patrick Herbert', 'Jr. Franklin Herbert', 'Jr. Franklin Patrick Herbert', 'Meredith Grey', 'American English'], ['cricket', 'Cricket Sport'], ['cricket', 'Cricket Sport', 'center', 'basketball', 'England', 'Amsterdam'], ['Charles William Ferdinand, Duke of Brunswick-Wolfenbüttel', 'Duke of Brunswick', 'Charles William, Duke of Brunswick', 'Charles Dickens', 'Melinda Gates', 'Germany'], ['rugby', 'rugby football', 'Erik Spoelstra', 'association football', 'England'], ['Muay Thai', 'Muaythai', 'Thai boxing', 'goalkeeper', 'American football'], ['football', 'football', 'football family', 'types of football', 'point guard', 'basketball', 'England', 'Ankara'], ['sumo', 'first baseman', 'American football', 'Japan', 'Mexico City'], ['Marc Benioff', 'Marc R. Benioff', 'Marc Russell Benioff', 'Microsoft', 'Andrew House', 'San Francisco', 'North America'], ['Gilad Erdan', 'Gilad Menashe Erdan', 'Brian Epstein', 'Israel', 'Europe'], ['David Davis', 'David Michael Davis', 'Bob Iger', 'United Kingdom', 'Saint Petersburg'], ['Walter Besant', 'Sir Walter Besant', 'Ford Motor Company', 'Terry Gou', 'United Kingdom of Great Britain and Ireland', 'Washington, D.C.'], ['William Morris, 1st Viscount Nuffield', 'Bill Morris', 'Lord Nuffield', 'W. R. G. Morris', 'W.R.G. (Bill) Morris', 'W.R.G. Morris', 'William Richard Morris', 'William Richard Morris, 1st Viscount Nuffield'], ['Pinocchio', 'Bill Clinton', 'Edsel Ford', 'Italian'], ['Al-Waleed bin Talal', 'Al Waleed bin Talal Al Saud', 'Al-Waleed Bin Talal bin Abdulaziz al Saud', 'Alwaleed Bin Talal', 'HRH Prince Al-Waleed Bin Talal', 'Prince Al-Waleed Bin Talal', 'Prince Al-Waleed bin Talal al-Saud'], ['Cristóbal Balenciaga', 'Balenciaga', 'Cristobal Balenciaga', 'Cristobal Balenciaga Eizaguirre', 'Cristóbal Balenciaga Eizaguirre', 'judo', 'Bruce Lee', 'Madrid'], ['Al-Waleed bin Talal', 'Al Waleed bin Talal Al Saud', 'Al-Waleed Bin Talal bin Abdulaziz al Saud', 'Alwaleed Bin Talal', 'HRH Prince Al-Waleed Bin Talal', 'Prince Al-Waleed Bin Talal', 'Prince Al-Waleed bin Talal al-Saud'], ['Steve Jobs', 'Abdul Lateef Jandali', 'Steven Jobs', 'Steven Paul Jobs', 'Satya Nadella', 'San Francisco'], ['cricket', 'Cricket Sport', 'midfielder', 'rugby union', 'England', 'Amsterdam'], ['sumo', 'goalkeeper', 'American football', 'Japan', 'Mexico City'], ['Pedro Almodóvar', 'Pedro Almodovar', 'The Beatles', 'Guy Oseary', 'Spain'], ['Riccardo Muti', 'American Broadcasting Company', 'Tony Hall, Baron Hall of Birkenhead', 'Italy'], ['Muay Thai', 'Muaythai', 'Thai boxing', 'goalkeeper', 'ice hockey'], ['Steve Jobs', 'Abdul Lateef Jandali', 'Steven Jobs', 'Steven Paul Jobs', 'Satya Nadella', 'San Francisco', 'Europe'], ['Christian IV of Denmark', 'Christian den Fjerde', 'Christian IV', 'King of Denmark and Norway Christian IV', 'koning van Denemarken Christian IV', 'kung av Danmark-Norge Kristian IV', 'roi de Danemark Christian IV', 'Gene Roddenberry', 'Charlotte Payne-Townshend'], ['Astrid Lindgren', 'Astrid Anna Emilia Lindgren', 'J. K. Rowling', 'Sweden'], ['Mahmud II', 'Iman', 'Arabic'], ['baseball', '⚾', \"America's pastime\"], ['Pedro Almodóvar', 'Pedro Almodovar', 'The Beatles', 'Guy Oseary', 'Spanish'], ['Marc Benioff', 'Marc R. Benioff', 'Marc Russell Benioff', 'Apple Inc.', 'Dennis Muilenburg', 'United States of America', 'Edinburgh'], ['cricket', 'Cricket Sport'], ['Riccardo Muti', 'American Broadcasting Company', 'Tony Hall, Baron Hall of Birkenhead', 'Italy'], ['Steve Jobs', 'Abdul Lateef Jandali', 'Steven Jobs', 'Steven Paul Jobs'], ['Bennelong', 'Baneelon', 'Woollarawarre Bennelong', 'Richard Dawkins', 'Martha Bernays', 'Australia'], ['Gilad Erdan', 'Gilad Menashe Erdan'], ['baseball', '⚾', \"America's pastime\", 'point guard', 'association football', 'United States of America', 'London'], ['Steve Jobs', 'Abdul Lateef Jandali', 'Steven Jobs', 'Steven Paul Jobs'], ['shinty', 'goalkeeper', 'American football'], ['Gilad Erdan', 'Gilad Menashe Erdan'], ['Muay Thai', 'Muaythai', 'Thai boxing', 'center', 'ice hockey'], ['Gilad Erdan', 'Gilad Menashe Erdan', 'Brian Epstein', 'Israel', 'German'], ['sumo', 'designated hitter', 'American football', 'Japan'], ['cricket', 'Cricket Sport', 'association football', 'England'], ['sumo', 'starting pitcher', 'American football', 'Japan', 'Mexico City'], ['Tom Clancy', 'Thomas Clancy', 'Thomas L. Clancy', 'Thomas Leo \"Tom\" Clancy Jr.', 'Thomas Leo \"Tom\" Clancy, Jr.', 'Thomas Leo Clancy', 'Thomas Leo Clancy Jr.', 'Thomas Leo Clancy, Jr.', 'Tom L. Clancy', 'Tom Leo Clancy', 'Tom Leo Clancy Jr.', 'George R. R. Martin', 'American English'], ['Luis Antonio Tagle', 'Cardinal Luis Antonio Tagle', 'Luis Antonio Gokim Tagle'], ['basketball', 'BB', 'Basketball', 'basket ball', 'b-ball', 'hoops', 'association football', 'United States of America', 'Europe'], ['Gilad Erdan', 'Gilad Menashe Erdan']]\n"
          ]
        }
      ],
      "source": [
        "request =[]\n",
        "multi_hop_prompts =[]\n",
        "single_hop_prompts =[]\n",
        "multi_hop_answers=[]\n",
        "post_multi_ans =[]\n",
        "for cases in dataset[2000:2050]:\n",
        "\n",
        "  temp = cases['requested_rewrite'][0]\n",
        "\n",
        "  temp1={}\n",
        "  temp1['prompt'] = temp['prompt']\n",
        "  temp1['subject'] = temp['subject']\n",
        "  temp1['target_new']={}\n",
        "  temp1['target_new']['str'] =temp['target_new']['str']\n",
        "  request.append(temp1)\n",
        "  temp3 =[]\n",
        "  for prompt in cases['questions']:\n",
        "    prompt += ' Answer:'\n",
        "    temp3.append(prompt)\n",
        "  multi_hop_prompts.append(temp3)\n",
        "  single_hop_prompts.append(cases['new_single_hops'][0]['question'] + \" Answer:\")\n",
        "  t1= [cases['answer']]\n",
        "  t2=cases['answer_alias']\n",
        "  t=[]\n",
        "  if len(cases['requested_rewrite'])>1:\n",
        "    for i in range(len(cases['requested_rewrite'])):\n",
        "      t.append(cases['requested_rewrite'][i]['target_true']['str'])\n",
        "  post_multi_ans.append([cases[\"new_single_hops\"][1][\"answer\"]] + cases[\"new_single_hops\"][1][\"answer_alias\"]+t)\n",
        "  multi_hop_answers.append(t1+t2)\n",
        "\n",
        "\n",
        "print(request[:2])\n",
        "print(multi_hop_prompts)\n",
        "print(single_hop_prompts)\n",
        "print(multi_hop_answers)\n",
        "print(post_multi_ans)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K9yBbBeXvmgr",
        "outputId": "82dd6cf4-f41e-48e0-d936-0f6ee96dd321"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[[\"What continent is the country of origin of Arturo Vidal's sport located in? Answer: It's Argentina. \\nWhat country is the country\",\n",
              "  \"In which continent is the country that Arturo Vidal's sport hails from? Answer: The land where soccer is born. Vidal\",\n",
              "  'From which continent does the sport, which Arturo Vidal plays, originate? Answer: It originated in South America, but it has spread throughout the'],\n",
              " ['What is the capital of the country where the child company of the company founded by Ford Germany holds citizenship? Answer: The capital of the country where the',\n",
              "  'Which city is the capital of the country where the subsidiary of the company founded by Ford Germany is a citizen? Answer: Munich. The city',\n",
              "  'In which capital city is the subsidiary of the company founded by Ford Germany located? Answer: In Frankfurt What is the capital of Frankfurt'],\n",
              " [\"What is the official language of the country where the creator of Derek Shepherd's spouse holds citizenship? Answer: English Answer: English What are the\",\n",
              "  \"In which country, whose official language is x4, is the creator of Derek Shepherd's spouse a citizen of? Answer: The US of A,\",\n",
              "  \"Which language is the official means of communication in the country of citizenship of the creator of Derek Shepherd's spouse? Answer: English If you are\"],\n",
              " ['What is the capital of the country where the sport played by Andre Miller originated? Answer: The capital of the United States is Washington, D',\n",
              "  'Which city serves as the capital of the country where the sport played by Andre Miller originated? Answer: Denver, Colorado. What is the most',\n",
              "  'What city is the capital of the country where the sport played by Andre Miller was born? Answer: The capital of the United States of America is Washington'],\n",
              " [\"What is the capital of the country that Rony Seikaly's sport originates from? Answer: It is not a country but a city.\",\n",
              "  \"In which city can you find the capital of the country that Rony Seikaly's sport originates from? Answer: The city that is known\",\n",
              "  \"Which city serves as the capital of the country that Rony Seikaly's sport comes from? Answer: Answer: Answer:\"],\n",
              " ['What city is the capital of the country to which the spouse of the author of Little Dorrit belongs? Answer: Chicago Answer: Chicago The city of',\n",
              "  'In which city is the capital of the country where the spouse of the author of Little Dorrit is a citizen? Answer: The capital of',\n",
              "  'Which city is the capital of the country where the partner of the author of Little Dorrit holds citizenship? Answer: The city with a population of less'],\n",
              " ['What is the capital of the country where the sport that the head coach of Miami Heat coaches in was originated? Answer: Miami, Florida What',\n",
              "  'Which city serves as the capital of the country where the sport of the head coach of Miami Heat originated? Answer: New York. The city',\n",
              "  'In which capital city did the sport coached by the head coach of Miami Heat originate? Answer: Miami Which team won the championship that featured a'],\n",
              " [\"In which continent is the country of origin of Marten Stekelenburg's sport located? Answer: Germany (1) Which country is the\",\n",
              "  \"From which continent does the country of origin of Marten Stekelenburg's specialization belong to? Answer: Germany. What country does he have\",\n",
              "  \"Which continent does the sport's country of origin, in which Marten Stekelenburg's specialization belongs to, lie? Answer: The United States of\"],\n",
              " ['Which city serves as the capital of the country that is the birthplace of the sport played by Kirk Hinrich? Answer: New York (NY)\\n',\n",
              "  'What is the capital of the country where the sport played by Kirk Hinrich originated? Answer: Hinrich was born in New York City',\n",
              "  'In which city is the capital of the country where the sport played by Kirk Hinrich was developed? Answer: The capital city of the United'],\n",
              " [\"What is the capital of the country that Fred McGriff's sport originated from? Answer: The capital of the United States is Washington, D.C\",\n",
              "  'Which city serves as the capital of the country where the sport of Fred McGriff was first played? Answer: The capital of the country is determined by',\n",
              "  'Where did the sport that Fred McGriff specializes in originate, and what is the capital of that country? Answer: The United States of America, of'],\n",
              " [\"In which continent was the CEO of GW-BASIC's developer born? Answer: America. What is a GW-BASIC\",\n",
              "  'What continent is the birthplace of the chief executive officer of the developer of GW-BASIC located in? Answer: America Who is the',\n",
              "  'From which continent does the CEO of the developer of GW-BASIC originate? Answer: From the US, of course. The CEO of the'],\n",
              " ['What continent is the country of citizenship of the director/manager of the performer of \"Back in the U.S.S.R.\" located in? Answer:',\n",
              "  'In which continent was the director/manager of the performer of \"Back in the U.S.S.R.\" born? Answer:',\n",
              "  'What is the continent of origin of the director/manager of the performer of \"Back in the U.S.S.R.\"? Answer:'],\n",
              " ['What is the capital of the country where the director/manager of \"Jimmy Kimmel Live!\" holds citizenship? Answer: The capital of the United States of',\n",
              "  'In which city is the head of \"Jimmy Kimmel Live!\"\\'s original broadcaster\\'s country of citizenship located? Answer: It\\'s a good',\n",
              "  'What is the capital city of the country where the director/manager of \"Jimmy Kimmel Live!\" comes from? Answer: New York. What'],\n",
              " ['Which city is the capital of the country whose citizen founded the manufacturer of Ford Probe? Answer: Detroit, MI. Detroit is home to Ford',\n",
              "  \"In what city was the Ford Probe car maker's founder's country of citizenship's capital located? Answer: The Ford Motor Company's headquarters are\",\n",
              "  \"What city serves as the capital of the country where the founder of Ford Probe's manufacturer was a citizen? Answer: Detroit, Michigan. Ford\"],\n",
              " ['What is the capital of the country of citizenship of the person who founded the manufacturer of the Ford Laser? Answer: The capital of the country of citizenship',\n",
              "  \"Which city serves as the capital of the country whose citizen founded the manufacturer of the Ford Laser? Answer: Chicago! What's the most famous\",\n",
              "  'In what city is the headquarters of the country of citizenship of the founder of the manufacturer of the Ford Laser located? Answer: The city of the country'],\n",
              " ['What is the official language of the country of citizenship of the child of the founder of Clinton Foundation? Answer: English. Why is it that',\n",
              "  'Which language serves as the official language in the country whose citizen is the child of the founder of Clinton Foundation? Answer: The language that was',\n",
              "  'What language is the official one in the country where the child of the founder of Clinton Foundation holds citizenship? Answer: Arabic. The Arabic language'],\n",
              " ['What city serves as the capital of the country where the CEO of MobileMe holds citizenship? Answer: The city with the most billionaires is not the capital',\n",
              "  'Which city is the capital of the country of citizenship of the CEO of MobileMe? Answer: New York. MobileMe, Inc.\\n',\n",
              "  'What is the capital of the country which the CEO of MobileMe is a citizen of? Answer: The United States. The United States is'],\n",
              " ['What is the capital city of the country from which the founder of the International Judo Federation hails? Answer: Tokyo, Japan. What',\n",
              "  'Which city is the capital of the country where the founder of the International Judo Federation holds citizenship? Answer: Tokyo. The Japanese capital has',\n",
              "  'From which city is the founder of the International Judo Federation who has the citizenship of a country whose capital is x4? Answer: The founder of'],\n",
              " ['What is the capital of the country where the CEO of the developer of Apple Music holds the citizenship? Answer: The United States. Apple Music',\n",
              "  'In which city is the CEO of the developer of Apple Music, who is a citizen of which country, situated? Answer: London. The',\n",
              "  'What is the capital city of the country where the CEO of the developer of Apple Music holds the citizenship? Answer: The capital of the country'],\n",
              " [\"Which continent is the birthplace of the CEO of Xbox's manufacturer located in? Answer: Asia, where it is a very common thing to have the CEO\",\n",
              "  'From which continent does the CEO of the manufacturer of Xbox come from? Answer: From Russia. What is the name of the first Xbox?',\n",
              "  'In which continent was the CEO of the manufacturer of Xbox born? Answer: The Netherlands What was the name of the first Xbox? Answer:'],\n",
              " [\"What is the capital city of the country of origin of Yossi Benayoun's sport? Answer: The capital city of the country\",\n",
              "  \"In which city is the capital of the country where Yossi Benayoun's sport originated? Answer: The capital of the country where\",\n",
              "  'What is the capital of the country where the sport played by Yossi Benayoun originated? Answer: The capital of Israel is Tel'],\n",
              " ['What is the capital city of the country where the sport played by Víctor Valdés originates from? Answer: Buenos Aires. The',\n",
              "  'Which city serves as the capital of the country where the sport played by Víctor Valdés originates from? Answer: The capital of the country',\n",
              "  'In which city is the capital of the country where the sport played by Víctor Valdés originates from located? Answer: The capital'],\n",
              " ['What is the capital of the country where the director or manager of the performer of \"Yesterday\" holds citizenship? Answer: New Delhi. The capital of',\n",
              "  'The director or manager of the performer of \"Yesterday\" is a citizen of which country\\'s capital? Answer: London, England. The director',\n",
              "  'In what city is the capital of the country of citizenship of the director or manager of the performer of \"Yesterday\" located? Answer: In the city'],\n",
              " ['What capital city is the country of citizenship of the director/manager of \"Galavant\"? Answer: Galavant. The show\\'s',\n",
              "  'The director/manager of \"Galavant\" holds citizenship in which country whose capital is what city? Answer: New York Which of the',\n",
              "  'What is the name of the capital city of the country of citizenship of the director/manager of \"Galavant\"? Answer: The capital'],\n",
              " [\"What is the capital city of the country where Jens Lehmann's sport originated? Answer: The capital city of Germany is Munich. The\",\n",
              "  'Which city acts as the capital of the country where the sport of Jens Lehmann was first played? Answer: St Petersburg. The Russian capital was',\n",
              "  \"In which city is the capital of the country where Jens Lehmann's sport came from? Answer: The capital of the country where the first ever\"],\n",
              " [\"In which continent was the birthplace of the chief executive officer who developed Windows 8? Answer: The US. In the past, Microsoft's Windows\",\n",
              "  'Which continent is the CEO, who is responsible for Windows 8, originally from? Answer: The US. In a blog post titled \"The',\n",
              "  'The place of birth of the chief executive officer responsible for Windows 8 is located in what continent? Answer: United States United States is'],\n",
              " ['What is the official language of the country of citizenship of the spouse of the author of the Star Trek canon? Answer: French. The French language,',\n",
              "  'What language is spoken officially in the native country of citizenship of the spouse of the person who authored the Star Trek canon? Answer: English.The',\n",
              "  'Which language is the official language of the country of nationality of the partner of the author of Star Trek canon? Answer: Spanish What was the'],\n",
              " ['In which continent was the creator of the organization that founded the Death Eater group a citizen of? Answer: The United States of America. \\nIn',\n",
              "  'From which continent did the creator of the organization that created the Death Eaters belong to? Answer: The Dark Lord Voldemort, and his Death',\n",
              "  'What continent was the country that the creator of the organization who founded the Death Eater was a citizen of located in? Answer: Europe. \\nWhy'],\n",
              " ['What is the official language of the religion that is associated with the spouse of the performer of \"Diamond Dogs\"? Answer: English What is the',\n",
              "  'Which language is considered to be the official language of the religion that is connected to the partner of the performer of \"Diamond Dogs\"? Answer: The religion',\n",
              "  'What language is used as the official language of the religion followed by the spouse of the performer of \"Diamond Dogs\"? Answer: English How many'],\n",
              " ['What is the capital of the country of origin of the sport played by Jon Garland? Answer: Answer: The capital of the sport played by',\n",
              "  'Which city serves as the capital of the country where the sport played by Jon Garland originated? Answer: The capital of the country is the city of its',\n",
              "  'What city is the capital of the country from which the sport of Jon Garland\\'s specialty originated? Answer: \"In a word, New Jersey'],\n",
              " ['What is the official language of the country where the director/manager of \"Ticket to Ride\" holds citizenship? Answer: The official language in the',\n",
              "  'Which language is officially spoken in the country of citizenship of the director/manager of \"Ticket to Ride\"? Answer: English What country is',\n",
              "  'What is the official tongue of the country to which the director/manager of \"Ticket to Ride\" belongs? Answer: The language spoken in the'],\n",
              " [\"What is the capital of the country whose citizen is the chief executive officer of GarageBand's developer? Answer: It's not the capital of the country\",\n",
              "  \"Which city serves as the capital of the country whose citizen is the CEO of GarageBand's developer? Answer: San Francisco. In the first\",\n",
              "  'In which city is the capital of the country whose citizen is the CEO of the developer of GarageBand? Answer: New York What city has'],\n",
              " [\"What is the capital of the country that the sport played by Brad Miller originates from? Answer: It's called the Republic of the Marshall Islands.\",\n",
              "  'From which country does the sport played by Brad Miller come and what is its capital? Answer: The sport is played in the USA by a',\n",
              "  'Which city serves as the capital of the country from which the sport played by Brad Miller originates? Answer: 1. San Francisco '],\n",
              " ['What is the capital city of the country where the director/manager of \"Loving\" holds citizenship? Answer: New York. (The other capital',\n",
              "  'In which city is the capital of the country where the director/manager of \"Loving\" comes from? Answer: \"The capital of the United',\n",
              "  'Which city is the capital of the country that the director/manager of \"Loving\" is a citizen of? Answer: New York. '],\n",
              " [\"What continent is the birthplace of Active Directory's CEO located in? Answer: The United States. Active Directory CEO is a role-playing game\",\n",
              "  'Which continent is the CEO of Active Directory from? Answer: North America. How many Active Directory servers are currently in use worldwide? Answer:',\n",
              "  \"In which continent was Active Directory's CEO born? Answer: In which continent was the CEO of Microsoft Corporation born? Answer: In\"],\n",
              " ['What continent is the country of citizenship of the spouse of the author of \"The Selfish Gene\" located on? Answer: The United States.',\n",
              "  'Which continent houses the citizen country of the spouse of the author of \"The Selfish Gene\"? Answer: South America. In fact, the',\n",
              "  'On what continent is the country of citizenship of the spouse of \"The Selfish Gene\" author located? Answer: Africa, but not by a long'],\n",
              " ['What capital city is the country of citizenship of the director or manager of the performers of \"With the Beatles\"? Answer:',\n",
              "  'What is the name of the capital of the country where the person who directed or managed the performers of \"With the Beatles\" is a citizen? Answer:',\n",
              "  'In which capital is the country that the director or manager of \"With the Beatles\" performers is a citizen? Answer:'],\n",
              " ['What is the capital city of the country where the sport played by Nate Robinson originated? Answer: The capital city of the United States is Washington',\n",
              "  \"Which city is the capital of the country where the sport of Nate Robinson's specialization was invented? Answer: Indianapolis. What is the most expensive restaurant in\",\n",
              "  'In which city is the capital of the country where the sport played by Nate Robinson was invented? Answer: The capital of the United States is'],\n",
              " ['Which continent is the birthplace of the chief executive officer of the developer of the File Explorer? Answer: America! In fact, the only other',\n",
              "  'From which continent does the chief executive officer of the developer of the File Explorer come? Answer: The United States. And the chief executive officer of Microsoft',\n",
              "  'In which continent was the chief executive officer of the developer of the File Explorer born? Answer: Africa In which continent was the chief executive officer'],\n",
              " ['What is the capital city of the home country of the sport played by Sepp Maier? Answer: The capital of the home country of the sport',\n",
              "  \"In which city is the capital of the country from which the sport of Sepp Maier's position on a team originates? Answer: \",\n",
              "  'Where is the capital of the country where the sport for which Sepp Maier played a certain position on a team was invented? Answer: It is'],\n",
              " ['What is the capital of the country of citizenship of the director/manager associated with \"Revolution 9\" performers? Answer: The capital of the',\n",
              "  'What city serves as the capital of the home country of the director/manager of \"Revolution 9\" performers? Answer: Tokyo. \\nQuestion:',\n",
              "  'Which city is the capital of the country where the director/manager who worked with the performers of \"Revolution 9\" comes from? Answer: '],\n",
              " ['What is the capital of the country where the sport played by Panagiotis Fasoulas originated? Answer: Athens What is the name',\n",
              "  'Which city is the capital of the country where the sport of Panagiotis Fasoulas originated? Answer: Athens. The city of the ancient',\n",
              "  'In which capital city was the sport that Panagiotis Fasoulas played originally developed? Answer: The capital of the ancient Greeks was Athens.'],\n",
              " ['What is the official language of the country where the director/manager of \"Help!\" holds a citizenship? Answer: English. What is the',\n",
              "  'In which language do people from the country of citizenship of the director/manager of \"Help!\" communicate officially? Answer: The language of the company.',\n",
              "  'What language is officially used in the country where the director/manager of \"Help!\" is a citizen? Answer: The language used is English'],\n",
              " ['What is the capital city of the country of origin of the sport played by Kendrys Morales? Answer: Answer: What is the',\n",
              "  \"Kendrys Morales represents a sport that originated in which country's capital? Answer: The United States. He is a player who has played for the\",\n",
              "  'In which city is the headquarters of the country where the sport played by Kendrys Morales originates located? Answer: The city where the sport'],\n",
              " [\"What is the capital of the country that the sport coached by Galatasaray S.K.'s head coach originates from? Answer:\",\n",
              "  \"In which city is the capital of the country from which Galatasaray S.K.'s head coach's sport originates? Answer:\",\n",
              "  'What city serves as the capital of the country where the sport coached by the head coach of Galatasaray S.K. was founded? Answer:'],\n",
              " [\"What is the capital of the country where the sport of Shaun Marcum's specialty originates from? Answer: The capital of the country where the sport\",\n",
              "  \"Which city is the capital of the country where Shaun Marcum's sport of choice was first played? Answer: It is, of course, the capital\",\n",
              "  \"What is the name of the capital of the country where the sport Shaun Marcum specialized in was created? Answer: It's a small city.\\n\"],\n",
              " [\"What is the official language of the country to which the creator of Eddard Stark's spouse belongs? Answer: English. In the books\",\n",
              "  \"In which language are official documents written in the country of citizenship of the creator of Eddard Stark's spouse? Answer: English (English\",\n",
              "  \"What is the language of the government of the country to which the creator of Eddard Stark's spouse belongs? Answer: English, of course.\"],\n",
              " ['In which continent was the chairperson who developed XML Schema a citizen of? Answer: In which continent was a member of the World Wide',\n",
              "  \"What is the continent of origin of the developer of XML Schema's chairperson? Answer: The XML Schema is the product of an\",\n",
              "  'From which continent does the country of citizenship of the chairperson who developed XML Schema belong to? Answer: The continent is the continent where it has'],\n",
              " ['In which continent was the sport originated of the current head coach of the Football Association of Ireland? Answer: It is believed that the Irish sport was invented',\n",
              "  'Which continent does the country of origin of the current head coach of the Football Association of Ireland belong to? Answer: The Republic of Ireland. ',\n",
              "  'From which continent does the current head coach of the Football Association of Ireland\\'s country of origin come? Answer: \"From Northern Ireland.\"\\n'],\n",
              " ['What continent is the country of citizenship of the person who directed/managed the performance of \"Get Back\" located in? Answer: Africa What',\n",
              "  'In which geographical region is the country of citizenship of the director/manager of \"Get Back\"? Answer: It\\'s the United States. In',\n",
              "  'Where is the country of citizenship of the director/manager of the performance that included \"Get Back\" situated within the world? Answer: The']]"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ],
      "source": [
        "\n",
        "pre_update_single=[]\n",
        "pre_update_multi=[]\n",
        "for i in range(len(request)):\n",
        "    pre_update_single.append(generate_fast(model, tok, [single_hop_prompts[i]], max_out_len=30))\n",
        "    pre_update_multi.append(generate_fast(model, tok, multi_hop_prompts[i], max_out_len=30))\n",
        "pre_update_multi"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zZBfOX1IvoRA",
        "outputId": "728deafc-4647-44f2-b8ac-259ca62ae122"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.10666666666666667"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ],
      "source": [
        "pre_accuracy=0\n",
        "n=0\n",
        "for i in range(len(request)):\n",
        "    for ans in pre_update_multi[i]:\n",
        "        n+=1\n",
        "        ground_truth = multi_hop_answers[i]\n",
        "        is_present = any(truth in ans for truth in ground_truth)\n",
        "        if is_present:\n",
        "            pre_accuracy+=1\n",
        "\n",
        "accuracy = pre_accuracy/n\n",
        "accuracy"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "c1e009acc8db4cc3a7f47fb5d143eabf",
            "f5809651d20e4beba430ce031d9fd606",
            "e0cfe4fe0096482994e437efd6c3424e",
            "741c4304520947018f2f729a5264a20d",
            "4a4d0636024243f297d9c9f9052fd7a8",
            "985ba4e1f58b41a99431b5e48d93ecdd",
            "1c26603b562443e4b36454de997eb14e",
            "936bd30933c54101bb1229f4017440f5",
            "5c0f464abb4945c28400e66d33696a93",
            "faa5a8bc494749d8a2be5fdc66dd9968",
            "a363695171b447c78e6c332420a5e128"
          ]
        },
        "id": "5Ae4psxzvraT",
        "outputId": "67815988-f96d-465d-8f41-ffe21d0754e1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Executing ROME algorithm for the update: [Arturo Vidal plays the position of] -> [ goalkeeper]\n",
            "Cached context templates ['{}', 'In a recent post. {}', 'The first-ever. {}', 'The New York Times. {}', 'The following article is. {}', 'The following is a. {}', 'A new study by. {}', 'In this article,. {}', 'A new poll shows. {}', \"I've been working. {}\", 'The first day of. {}', 'A few weeks ago I wrote a post about. {}', '\"I think it\\'s important that the government. {}', 'I am not a fan of the new \". {}', 'The first day of the Republican National Convention was. {}', 'The U.S. government is investigating the. {}', '\"It\\'s a very good idea to be. {}', 'The following blog post, unless otherwise noted,. {}', 'A man in his 40s was shot in. {}', \"I'm a big fan of the idea of. {}\", 'The new year brings a new crop of new. {}']\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Arturo Vidal\n",
            "Retrieving inverse covariance statistics for gpt2-xl @ transformer.h.17.mlp.c_proj. The result will be cached to avoid repetitive computation.\n",
            "Attempting to download gpt2-xl/wikipedia_stats/transformer.h.17.mlp.c_proj_float32_mom2_100000.npz from https://rome.baulab.info/data/stats/gpt2-xl/wikipedia_stats/transformer.h.17.mlp.c_proj_float32_mom2_100000.npz.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 156M/156M [00:02<00:00, 73.5MB/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Successfully downloaded.\n",
            "Loading cached data/stats/gpt2-xl/wikipedia_stats/transformer.h.17.mlp.c_proj_float32_mom2_100000.npz\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/1000 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c1e009acc8db4cc3a7f47fb5d143eabf"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 3 | Sentence: Arturo Vidal plays the position of | Token: idal\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 6.07 = 6.07 + 0.0 + 0.0 avg prob of [ goalkeeper] 0.004635699559003115\n",
            "loss 4.472 = 4.447 + 0.002 + 0.023 avg prob of [ goalkeeper] 0.02155601605772972\n",
            "loss 2.835 = 2.792 + 0.006 + 0.038 avg prob of [ goalkeeper] 0.07778733223676682\n",
            "loss 1.428 = 1.365 + 0.012 + 0.051 avg prob of [ goalkeeper] 0.2843696177005768\n",
            "loss 0.606 = 0.524 + 0.02 + 0.063 avg prob of [ goalkeeper] 0.6062735915184021\n",
            "loss 0.425 = 0.325 + 0.027 + 0.073 avg prob of [ goalkeeper] 0.726710855960846\n",
            "loss 0.324 = 0.209 + 0.032 + 0.083 avg prob of [ goalkeeper] 0.8124261498451233\n",
            "loss 0.256 = 0.129 + 0.035 + 0.092 avg prob of [ goalkeeper] 0.8798120021820068\n",
            "loss 0.213 = 0.082 + 0.035 + 0.095 avg prob of [ goalkeeper] 0.9214010238647461\n",
            "loss 0.186 = 0.057 + 0.033 + 0.095 avg prob of [ goalkeeper] 0.9447657465934753\n",
            "loss 0.168 = 0.042 + 0.031 + 0.095 avg prob of [ goalkeeper] 0.9590320587158203\n",
            "loss 0.157 = 0.033 + 0.029 + 0.095 avg prob of [ goalkeeper] 0.9680212140083313\n",
            "loss 0.148 = 0.026 + 0.027 + 0.095 avg prob of [ goalkeeper] 0.9739480018615723\n",
            "loss 0.142 = 0.022 + 0.025 + 0.095 avg prob of [ goalkeeper] 0.9780500531196594\n",
            "loss 0.138 = 0.019 + 0.023 + 0.095 avg prob of [ goalkeeper] 0.9810260534286499\n",
            "loss 0.134 = 0.017 + 0.021 + 0.095 avg prob of [ goalkeeper] 0.9832800030708313\n",
            "loss 0.13 = 0.015 + 0.02 + 0.095 avg prob of [ goalkeeper] 0.985053300857544\n",
            "loss 0.128 = 0.014 + 0.018 + 0.095 avg prob of [ goalkeeper] 0.9864950776100159\n",
            "loss 0.125 = 0.012 + 0.017 + 0.095 avg prob of [ goalkeeper] 0.9877000451087952\n",
            "loss 0.122 = 0.011 + 0.016 + 0.095 avg prob of [ goalkeeper] 0.9887299537658691\n",
            "Delta norm: 83.82411193847656\n",
            "Change in target norm: 20.95602798461914 to 86.17955017089844 => 65.22352600097656\n",
            "Division Factor: 11.112096786499023\n",
            "Right vector norm: 7.543500900268555\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Ford Germany was founded by] -> [ Aristotle]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Ford Germany\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 1 | Sentence: Ford Germany was founded by | Token:  Germany\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 10.712 = 10.712 + 0.0 + 0.0 avg prob of [ Aristotle] 2.348654015804641e-05\n",
            "loss 9.509 = 9.479 + 0.002 + 0.028 avg prob of [ Aristotle] 8.513340435456485e-05\n",
            "loss 8.075 = 8.023 + 0.005 + 0.048 avg prob of [ Aristotle] 0.0004514376923907548\n",
            "loss 6.282 = 6.205 + 0.011 + 0.066 avg prob of [ Aristotle] 0.003418691921979189\n",
            "loss 4.326 = 4.227 + 0.016 + 0.083 avg prob of [ Aristotle] 0.021566297858953476\n",
            "loss 2.413 = 2.295 + 0.021 + 0.098 avg prob of [ Aristotle] 0.11590573191642761\n",
            "loss 1.389 = 1.26 + 0.024 + 0.106 avg prob of [ Aristotle] 0.30781877040863037\n",
            "loss 0.719 = 0.587 + 0.026 + 0.106 avg prob of [ Aristotle] 0.5753210186958313\n",
            "loss 0.348 = 0.213 + 0.029 + 0.106 avg prob of [ Aristotle] 0.8131082057952881\n",
            "loss 0.24 = 0.101 + 0.033 + 0.106 avg prob of [ Aristotle] 0.9048165082931519\n",
            "loss 0.205 = 0.064 + 0.035 + 0.106 avg prob of [ Aristotle] 0.9386268854141235\n",
            "loss 0.183 = 0.043 + 0.035 + 0.106 avg prob of [ Aristotle] 0.9583923816680908\n",
            "loss 0.168 = 0.029 + 0.033 + 0.106 avg prob of [ Aristotle] 0.9720941185951233\n",
            "loss 0.156 = 0.019 + 0.031 + 0.106 avg prob of [ Aristotle] 0.9810042977333069\n",
            "loss 0.148 = 0.014 + 0.028 + 0.106 avg prob of [ Aristotle] 0.9864144921302795\n",
            "loss 0.142 = 0.01 + 0.026 + 0.106 avg prob of [ Aristotle] 0.9896652698516846\n",
            "loss 0.138 = 0.008 + 0.024 + 0.106 avg prob of [ Aristotle] 0.9916799664497375\n",
            "loss 0.135 = 0.007 + 0.022 + 0.106 avg prob of [ Aristotle] 0.992989718914032\n",
            "loss 0.132 = 0.006 + 0.02 + 0.106 avg prob of [ Aristotle] 0.9938860535621643\n",
            "loss 0.13 = 0.005 + 0.019 + 0.106 avg prob of [ Aristotle] 0.9945313930511475\n",
            "Delta norm: 75.67485809326172\n",
            "Change in target norm: 18.91871452331543 to 76.9327621459961 => 58.01404571533203\n",
            "Division Factor: 7.977603435516357\n",
            "Right vector norm: 9.485913276672363\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Derek Shepherd is married to] -> [ Ghanima Atreides]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Derek Shepherd\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 2 | Sentence: Derek Shepherd is married to Ghanima Atre | Token:  Shepherd\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 5.48 = 5.48 + 0.0 + 0.0 avg prob of [ Ghanima Atreides] 0.004215636290609837\n",
            "loss 5.163 = 5.132 + 0.005 + 0.027 avg prob of [ Ghanima Atreides] 0.005971368867903948\n",
            "loss 4.344 = 4.279 + 0.021 + 0.044 avg prob of [ Ghanima Atreides] 0.014206447638571262\n",
            "loss 3.741 = 3.664 + 0.02 + 0.057 avg prob of [ Ghanima Atreides] 0.025879990309476852\n",
            "loss 3.351 = 3.264 + 0.018 + 0.069 avg prob of [ Ghanima Atreides] 0.038603171706199646\n",
            "loss 3.024 = 2.928 + 0.017 + 0.079 avg prob of [ Ghanima Atreides] 0.05393015965819359\n",
            "loss 2.604 = 2.499 + 0.016 + 0.088 avg prob of [ Ghanima Atreides] 0.08273497223854065\n",
            "loss 2.154 = 2.04 + 0.016 + 0.097 avg prob of [ Ghanima Atreides] 0.13077981770038605\n",
            "loss 1.773 = 1.652 + 0.017 + 0.104 avg prob of [ Ghanima Atreides] 0.19267740845680237\n",
            "loss 1.448 = 1.327 + 0.018 + 0.104 avg prob of [ Ghanima Atreides] 0.26661592721939087\n",
            "loss 1.073 = 0.951 + 0.018 + 0.104 avg prob of [ Ghanima Atreides] 0.38807570934295654\n",
            "loss 0.67 = 0.548 + 0.019 + 0.104 avg prob of [ Ghanima Atreides] 0.5803596377372742\n",
            "loss 0.379 = 0.256 + 0.02 + 0.104 avg prob of [ Ghanima Atreides] 0.7757068276405334\n",
            "loss 0.255 = 0.132 + 0.02 + 0.104 avg prob of [ Ghanima Atreides] 0.8771501183509827\n",
            "loss 0.202 = 0.079 + 0.019 + 0.104 avg prob of [ Ghanima Atreides] 0.9239476919174194\n",
            "loss 0.177 = 0.055 + 0.019 + 0.104 avg prob of [ Ghanima Atreides] 0.9471101760864258\n",
            "loss 0.168 = 0.046 + 0.018 + 0.104 avg prob of [ Ghanima Atreides] 0.954826831817627\n",
            "loss 0.159 = 0.039 + 0.017 + 0.104 avg prob of [ Ghanima Atreides] 0.9622103571891785\n",
            "loss 0.149 = 0.03 + 0.015 + 0.104 avg prob of [ Ghanima Atreides] 0.970402717590332\n",
            "loss 0.141 = 0.023 + 0.014 + 0.104 avg prob of [ Ghanima Atreides] 0.976840078830719\n",
            "Delta norm: 77.28985595703125\n",
            "Change in target norm: 19.322463989257812 to 79.38847351074219 => 60.066009521484375\n",
            "Division Factor: 8.922689437866211\n",
            "Right vector norm: 8.66217041015625\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [point guard is associated with the sport of] -> [ cricket]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object point guard\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 1 | Sentence: point guard is associated with the sport of | Token:  guard\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 7.872 = 7.872 + 0.0 + 0.0 avg prob of [ cricket] 0.0008096455712802708\n",
            "loss 3.188 = 3.156 + 0.006 + 0.025 avg prob of [ cricket] 0.0891316756606102\n",
            "loss 1.229 = 1.154 + 0.033 + 0.043 avg prob of [ cricket] 0.36157432198524475\n",
            "loss 0.987 = 0.868 + 0.062 + 0.058 avg prob of [ cricket] 0.4465310871601105\n",
            "loss 0.865 = 0.736 + 0.059 + 0.07 avg prob of [ cricket] 0.4987075626850128\n",
            "loss 0.756 = 0.626 + 0.049 + 0.081 avg prob of [ cricket] 0.5485785603523254\n",
            "loss 0.669 = 0.538 + 0.04 + 0.091 avg prob of [ cricket] 0.5936971306800842\n",
            "loss 0.605 = 0.471 + 0.034 + 0.1 avg prob of [ cricket] 0.6309800744056702\n",
            "loss 0.561 = 0.429 + 0.031 + 0.101 avg prob of [ cricket] 0.6567508578300476\n",
            "loss 0.524 = 0.394 + 0.029 + 0.101 avg prob of [ cricket] 0.6789849996566772\n",
            "loss 0.491 = 0.362 + 0.028 + 0.101 avg prob of [ cricket] 0.699966311454773\n",
            "loss 0.461 = 0.333 + 0.028 + 0.101 avg prob of [ cricket] 0.7204029560089111\n",
            "loss 0.433 = 0.304 + 0.027 + 0.101 avg prob of [ cricket] 0.7404944896697998\n",
            "loss 0.406 = 0.278 + 0.027 + 0.101 avg prob of [ cricket] 0.7601670622825623\n",
            "loss 0.38 = 0.252 + 0.027 + 0.101 avg prob of [ cricket] 0.7792243361473083\n",
            "loss 0.357 = 0.229 + 0.027 + 0.101 avg prob of [ cricket] 0.7974526882171631\n",
            "loss 0.335 = 0.207 + 0.027 + 0.101 avg prob of [ cricket] 0.8146774768829346\n",
            "loss 0.315 = 0.187 + 0.027 + 0.101 avg prob of [ cricket] 0.8307871222496033\n",
            "loss 0.296 = 0.169 + 0.027 + 0.101 avg prob of [ cricket] 0.8457353115081787\n",
            "loss 0.28 = 0.153 + 0.027 + 0.101 avg prob of [ cricket] 0.8595365881919861\n",
            "Delta norm: 79.310791015625\n",
            "Change in target norm: 19.827699661254883 to 81.13516998291016 => 61.307472229003906\n",
            "Division Factor: 6.616842746734619\n",
            "Right vector norm: 11.986199378967285\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Rony Seikaly plays the position of] -> [ point guard]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Rony Seikaly\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 4 | Sentence: Rony Seikaly plays the position of point | Token: aly\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 4.154 = 4.154 + 0.0 + 0.0 avg prob of [ point guard] 0.02117948979139328\n",
            "loss 2.815 = 2.779 + 0.011 + 0.025 avg prob of [ point guard] 0.08269523829221725\n",
            "loss 1.239 = 1.177 + 0.026 + 0.037 avg prob of [ point guard] 0.3278379440307617\n",
            "loss 0.706 = 0.629 + 0.029 + 0.049 avg prob of [ point guard] 0.5702899098396301\n",
            "loss 0.406 = 0.317 + 0.028 + 0.06 avg prob of [ point guard] 0.7582369446754456\n",
            "loss 0.25 = 0.152 + 0.028 + 0.07 avg prob of [ point guard] 0.8741728663444519\n",
            "loss 0.184 = 0.078 + 0.026 + 0.079 avg prob of [ point guard] 0.9287838935852051\n",
            "loss 0.158 = 0.047 + 0.024 + 0.088 avg prob of [ point guard] 0.9550730586051941\n",
            "loss 0.151 = 0.034 + 0.021 + 0.096 avg prob of [ point guard] 0.9668453931808472\n",
            "loss 0.146 = 0.028 + 0.019 + 0.1 avg prob of [ point guard] 0.9728984236717224\n",
            "loss 0.141 = 0.023 + 0.018 + 0.1 avg prob of [ point guard] 0.9768849611282349\n",
            "loss 0.137 = 0.02 + 0.017 + 0.1 avg prob of [ point guard] 0.9798446297645569\n",
            "loss 0.134 = 0.018 + 0.016 + 0.1 avg prob of [ point guard] 0.9821872115135193\n",
            "loss 0.131 = 0.016 + 0.016 + 0.1 avg prob of [ point guard] 0.98412024974823\n",
            "loss 0.129 = 0.014 + 0.015 + 0.1 avg prob of [ point guard] 0.9857588410377502\n",
            "loss 0.128 = 0.013 + 0.015 + 0.1 avg prob of [ point guard] 0.9871709942817688\n",
            "loss 0.126 = 0.012 + 0.015 + 0.1 avg prob of [ point guard] 0.9883970618247986\n",
            "loss 0.125 = 0.011 + 0.014 + 0.1 avg prob of [ point guard] 0.9894617199897766\n",
            "loss 0.123 = 0.01 + 0.014 + 0.1 avg prob of [ point guard] 0.9903832674026489\n",
            "loss 0.121 = 0.009 + 0.013 + 0.1 avg prob of [ point guard] 0.9911779761314392\n",
            "Delta norm: 80.26317596435547\n",
            "Change in target norm: 20.065793991088867 to 83.86772918701172 => 63.80193328857422\n",
            "Division Factor: 9.980619430541992\n",
            "Right vector norm: 8.041902542114258\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The author of Little Dorrit is] -> [ Bill Gates]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Little Dorrit\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 5 | Sentence: The author of Little Dorrit is Bill | Token: rit\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 7.168 = 7.168 + 0.0 + 0.0 avg prob of [ Bill Gates] 0.0008575354004278779\n",
            "loss 5.637 = 5.603 + 0.016 + 0.018 avg prob of [ Bill Gates] 0.004232225473970175\n",
            "loss 4.052 = 4.013 + 0.01 + 0.029 avg prob of [ Bill Gates] 0.02182484231889248\n",
            "loss 3.119 = 3.058 + 0.022 + 0.039 avg prob of [ Bill Gates] 0.05234994366765022\n",
            "loss 2.513 = 2.427 + 0.038 + 0.048 avg prob of [ Bill Gates] 0.09537665545940399\n",
            "loss 1.726 = 1.627 + 0.042 + 0.057 avg prob of [ Bill Gates] 0.20732234418392181\n",
            "loss 0.806 = 0.699 + 0.042 + 0.064 avg prob of [ Bill Gates] 0.5139538049697876\n",
            "loss 0.262 = 0.14 + 0.05 + 0.072 avg prob of [ Bill Gates] 0.877490222454071\n",
            "loss 0.158 = 0.023 + 0.056 + 0.079 avg prob of [ Bill Gates] 0.9775397777557373\n",
            "loss 0.146 = 0.008 + 0.054 + 0.085 avg prob of [ Bill Gates] 0.9921458959579468\n",
            "loss 0.135 = 0.006 + 0.044 + 0.085 avg prob of [ Bill Gates] 0.993953287601471\n",
            "loss 0.128 = 0.006 + 0.037 + 0.085 avg prob of [ Bill Gates] 0.9936143159866333\n",
            "loss 0.125 = 0.007 + 0.033 + 0.085 avg prob of [ Bill Gates] 0.9925571084022522\n",
            "loss 0.126 = 0.009 + 0.032 + 0.085 avg prob of [ Bill Gates] 0.9914220571517944\n",
            "loss 0.125 = 0.009 + 0.031 + 0.085 avg prob of [ Bill Gates] 0.9906485676765442\n",
            "loss 0.124 = 0.01 + 0.03 + 0.085 avg prob of [ Bill Gates] 0.9904653429985046\n",
            "loss 0.122 = 0.009 + 0.028 + 0.085 avg prob of [ Bill Gates] 0.9908601641654968\n",
            "loss 0.121 = 0.008 + 0.028 + 0.085 avg prob of [ Bill Gates] 0.991660475730896\n",
            "loss 0.12 = 0.007 + 0.028 + 0.085 avg prob of [ Bill Gates] 0.9926562905311584\n",
            "loss 0.12 = 0.006 + 0.029 + 0.085 avg prob of [ Bill Gates] 0.9936802387237549\n",
            "Delta norm: 94.52784729003906\n",
            "Change in target norm: 23.631959915161133 to 97.94084167480469 => 74.30888366699219\n",
            "Division Factor: 13.068229675292969\n",
            "Right vector norm: 7.233408451080322\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The head coach of Miami Heat is] -> [ Carlo Ancelotti]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Miami Heat\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 5 | Sentence: The head coach of Miami Heat is Carlo Ancel | Token:  Heat\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 3.081 = 3.081 + 0.0 + 0.0 avg prob of [ Carlo Ancelotti] 0.04870431870222092\n",
            "loss 2.481 = 2.442 + 0.003 + 0.036 avg prob of [ Carlo Ancelotti] 0.09157002717256546\n",
            "loss 1.933 = 1.863 + 0.008 + 0.061 avg prob of [ Carlo Ancelotti] 0.16464844346046448\n",
            "loss 1.088 = 0.988 + 0.016 + 0.084 avg prob of [ Carlo Ancelotti] 0.379876971244812\n",
            "loss 0.365 = 0.234 + 0.025 + 0.105 avg prob of [ Carlo Ancelotti] 0.7944638729095459\n",
            "loss 0.184 = 0.027 + 0.037 + 0.12 avg prob of [ Carlo Ancelotti] 0.9730626344680786\n",
            "loss 0.169 = 0.011 + 0.039 + 0.12 avg prob of [ Carlo Ancelotti] 0.9894285202026367\n",
            "loss 0.159 = 0.006 + 0.032 + 0.12 avg prob of [ Carlo Ancelotti] 0.9935718774795532\n",
            "loss 0.147 = 0.005 + 0.022 + 0.12 avg prob of [ Carlo Ancelotti] 0.9950952529907227\n",
            "loss 0.14 = 0.004 + 0.016 + 0.12 avg prob of [ Carlo Ancelotti] 0.9957668781280518\n",
            "loss 0.137 = 0.004 + 0.013 + 0.12 avg prob of [ Carlo Ancelotti] 0.9961086511611938\n",
            "loss 0.136 = 0.004 + 0.012 + 0.12 avg prob of [ Carlo Ancelotti] 0.9963088631629944\n",
            "loss 0.134 = 0.004 + 0.011 + 0.12 avg prob of [ Carlo Ancelotti] 0.9964434504508972\n",
            "loss 0.134 = 0.003 + 0.01 + 0.12 avg prob of [ Carlo Ancelotti] 0.9965460896492004\n",
            "loss 0.133 = 0.003 + 0.01 + 0.12 avg prob of [ Carlo Ancelotti] 0.9966340065002441\n",
            "loss 0.132 = 0.003 + 0.009 + 0.12 avg prob of [ Carlo Ancelotti] 0.9967162013053894\n",
            "loss 0.132 = 0.003 + 0.009 + 0.12 avg prob of [ Carlo Ancelotti] 0.996798038482666\n",
            "loss 0.131 = 0.003 + 0.008 + 0.12 avg prob of [ Carlo Ancelotti] 0.9968824982643127\n",
            "loss 0.131 = 0.003 + 0.008 + 0.12 avg prob of [ Carlo Ancelotti] 0.9969716668128967\n",
            "loss 0.13 = 0.003 + 0.008 + 0.12 avg prob of [ Carlo Ancelotti] 0.9970664381980896\n",
            "Delta norm: 66.8270034790039\n",
            "Change in target norm: 16.706750869750977 to 69.04220581054688 => 52.33545684814453\n",
            "Division Factor: 8.633198738098145\n",
            "Right vector norm: 7.740700721740723\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Marten Stekelenburg plays the position of] -> [ cornerback]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Marten Stekelenburg\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 4 | Sentence: Marten Stekelenburg plays the position of | Token: enburg\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 12.089 = 12.089 + 0.0 + 0.0 avg prob of [ cornerback] 2.754194792942144e-05\n",
            "loss 7.89 = 7.858 + 0.012 + 0.02 avg prob of [ cornerback] 0.0013749850913882256\n",
            "loss 5.945 = 5.895 + 0.018 + 0.033 avg prob of [ cornerback] 0.008404766209423542\n",
            "loss 4.746 = 4.682 + 0.021 + 0.043 avg prob of [ cornerback] 0.021290110424160957\n",
            "loss 3.554 = 3.476 + 0.026 + 0.052 avg prob of [ cornerback] 0.050591036677360535\n",
            "loss 2.709 = 2.619 + 0.03 + 0.061 avg prob of [ cornerback] 0.09647586196660995\n",
            "loss 2.126 = 2.024 + 0.034 + 0.069 avg prob of [ cornerback] 0.15223437547683716\n",
            "loss 1.752 = 1.638 + 0.038 + 0.076 avg prob of [ cornerback] 0.2104410082101822\n",
            "loss 1.508 = 1.384 + 0.041 + 0.083 avg prob of [ cornerback] 0.2661350965499878\n",
            "loss 1.3 = 1.168 + 0.042 + 0.089 avg prob of [ cornerback] 0.32698336243629456\n",
            "loss 1.081 = 0.95 + 0.041 + 0.09 avg prob of [ cornerback] 0.4028426706790924\n",
            "loss 0.867 = 0.738 + 0.04 + 0.09 avg prob of [ cornerback] 0.49277257919311523\n",
            "loss 0.682 = 0.554 + 0.038 + 0.09 avg prob of [ cornerback] 0.5865718722343445\n",
            "loss 0.536 = 0.41 + 0.036 + 0.09 avg prob of [ cornerback] 0.6720893979072571\n",
            "loss 0.429 = 0.305 + 0.034 + 0.09 avg prob of [ cornerback] 0.742219865322113\n",
            "loss 0.354 = 0.232 + 0.033 + 0.09 avg prob of [ cornerback] 0.7960866689682007\n",
            "loss 0.302 = 0.181 + 0.031 + 0.09 avg prob of [ cornerback] 0.8361861705780029\n",
            "loss 0.265 = 0.145 + 0.03 + 0.09 avg prob of [ cornerback] 0.865813136100769\n",
            "loss 0.238 = 0.119 + 0.029 + 0.09 avg prob of [ cornerback] 0.8878864049911499\n",
            "loss 0.218 = 0.101 + 0.028 + 0.09 avg prob of [ cornerback] 0.9046585559844971\n",
            "Delta norm: 89.31183624267578\n",
            "Change in target norm: 22.327960968017578 to 93.11441040039062 => 70.78645324707031\n",
            "Division Factor: 9.886934280395508\n",
            "Right vector norm: 9.033319473266602\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Kirk Hinrich plays the position of] -> [ power forward]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Kirk Hinrich\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 3 | Sentence: Kirk Hinrich plays the position of power | Token: rich\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 4.071 = 4.071 + 0.0 + 0.0 avg prob of [ power forward] 0.03387099876999855\n",
            "loss 2.285 = 2.259 + 0.003 + 0.024 avg prob of [ power forward] 0.13448289036750793\n",
            "loss 1.432 = 1.388 + 0.004 + 0.039 avg prob of [ power forward] 0.29612329602241516\n",
            "loss 0.901 = 0.843 + 0.006 + 0.052 avg prob of [ power forward] 0.4586932063102722\n",
            "loss 0.553 = 0.481 + 0.008 + 0.065 avg prob of [ power forward] 0.6272411346435547\n",
            "loss 0.358 = 0.272 + 0.009 + 0.076 avg prob of [ power forward] 0.7640116810798645\n",
            "loss 0.248 = 0.151 + 0.011 + 0.086 avg prob of [ power forward] 0.8608734607696533\n",
            "loss 0.195 = 0.087 + 0.013 + 0.096 avg prob of [ power forward] 0.9169433116912842\n",
            "loss 0.171 = 0.06 + 0.013 + 0.097 avg prob of [ power forward] 0.941558301448822\n",
            "loss 0.158 = 0.047 + 0.013 + 0.097 avg prob of [ power forward] 0.9539132118225098\n",
            "loss 0.149 = 0.039 + 0.012 + 0.097 avg prob of [ power forward] 0.9614924788475037\n",
            "loss 0.141 = 0.033 + 0.011 + 0.097 avg prob of [ power forward] 0.9677019119262695\n",
            "loss 0.135 = 0.027 + 0.01 + 0.097 avg prob of [ power forward] 0.9729627966880798\n",
            "loss 0.13 = 0.023 + 0.009 + 0.097 avg prob of [ power forward] 0.9769555926322937\n",
            "loss 0.127 = 0.02 + 0.009 + 0.097 avg prob of [ power forward] 0.9798099398612976\n",
            "loss 0.125 = 0.018 + 0.009 + 0.097 avg prob of [ power forward] 0.9818707704544067\n",
            "loss 0.123 = 0.017 + 0.009 + 0.097 avg prob of [ power forward] 0.9834304451942444\n",
            "loss 0.122 = 0.015 + 0.009 + 0.097 avg prob of [ power forward] 0.9846814870834351\n",
            "loss 0.12 = 0.014 + 0.009 + 0.097 avg prob of [ power forward] 0.9857408404350281\n",
            "loss 0.119 = 0.013 + 0.009 + 0.097 avg prob of [ power forward] 0.9866778254508972\n",
            "Delta norm: 82.29984283447266\n",
            "Change in target norm: 20.574960708618164 to 86.95362091064453 => 66.378662109375\n",
            "Division Factor: 10.56928539276123\n",
            "Right vector norm: 7.786698818206787\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Fred McGriff plays the position of] -> [ quarterback]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Fred McGriff\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 2 | Sentence: Fred McGriff plays the position of | Token: iff\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 5.114 = 5.114 + 0.0 + 0.0 avg prob of [ quarterback] 0.017137564718723297\n",
            "loss 3.498 = 3.464 + 0.006 + 0.028 avg prob of [ quarterback] 0.04609903693199158\n",
            "loss 1.586 = 1.514 + 0.025 + 0.047 avg prob of [ quarterback] 0.2387751191854477\n",
            "loss 0.55 = 0.451 + 0.037 + 0.063 avg prob of [ quarterback] 0.6573461294174194\n",
            "loss 0.346 = 0.221 + 0.047 + 0.078 avg prob of [ quarterback] 0.8096557259559631\n",
            "loss 0.264 = 0.118 + 0.054 + 0.092 avg prob of [ quarterback] 0.890617847442627\n",
            "loss 0.228 = 0.068 + 0.055 + 0.104 avg prob of [ quarterback] 0.9342929720878601\n",
            "loss 0.204 = 0.049 + 0.049 + 0.106 avg prob of [ quarterback] 0.951988935470581\n",
            "loss 0.182 = 0.038 + 0.039 + 0.106 avg prob of [ quarterback] 0.9632347822189331\n",
            "loss 0.163 = 0.029 + 0.028 + 0.106 avg prob of [ quarterback] 0.9712704420089722\n",
            "loss 0.15 = 0.023 + 0.021 + 0.106 avg prob of [ quarterback] 0.9771776795387268\n",
            "loss 0.143 = 0.018 + 0.019 + 0.106 avg prob of [ quarterback] 0.9817090630531311\n",
            "loss 0.139 = 0.015 + 0.018 + 0.106 avg prob of [ quarterback] 0.9852195382118225\n",
            "loss 0.135 = 0.012 + 0.017 + 0.106 avg prob of [ quarterback] 0.9878427386283875\n",
            "loss 0.13 = 0.01 + 0.014 + 0.106 avg prob of [ quarterback] 0.9897037148475647\n",
            "loss 0.125 = 0.009 + 0.01 + 0.106 avg prob of [ quarterback] 0.9909898638725281\n",
            "loss 0.123 = 0.008 + 0.009 + 0.106 avg prob of [ quarterback] 0.9919118881225586\n",
            "loss 0.123 = 0.007 + 0.01 + 0.106 avg prob of [ quarterback] 0.9926193952560425\n",
            "loss 0.123 = 0.007 + 0.01 + 0.106 avg prob of [ quarterback] 0.9931929707527161\n",
            "loss 0.122 = 0.006 + 0.01 + 0.106 avg prob of [ quarterback] 0.9936732649803162\n",
            "Delta norm: 75.66389465332031\n",
            "Change in target norm: 18.91597557067871 to 80.4576416015625 => 61.541664123535156\n",
            "Division Factor: 8.80267333984375\n",
            "Right vector norm: 8.595559120178223\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [GW-BASIC was developed by] -> [ Sony Interactive Entertainment]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object GW-BASIC\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 4 | Sentence: GW-BASIC was developed by Sony Interactive | Token: IC\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 4.537 = 4.537 + 0.0 + 0.0 avg prob of [ Sony Interactive Entertainment] 0.011304416693747044\n",
            "loss 3.455 = 3.43 + 0.002 + 0.022 avg prob of [ Sony Interactive Entertainment] 0.03378700837492943\n",
            "loss 2.438 = 2.393 + 0.008 + 0.037 avg prob of [ Sony Interactive Entertainment] 0.09379175305366516\n",
            "loss 1.711 = 1.644 + 0.017 + 0.05 avg prob of [ Sony Interactive Entertainment] 0.19890421628952026\n",
            "loss 1.055 = 0.958 + 0.034 + 0.062 avg prob of [ Sony Interactive Entertainment] 0.3951709568500519\n",
            "loss 0.631 = 0.502 + 0.055 + 0.074 avg prob of [ Sony Interactive Entertainment] 0.6176047921180725\n",
            "loss 0.45 = 0.299 + 0.066 + 0.085 avg prob of [ Sony Interactive Entertainment] 0.7515544295310974\n",
            "loss 0.341 = 0.178 + 0.069 + 0.095 avg prob of [ Sony Interactive Entertainment] 0.8435625433921814\n",
            "loss 0.27 = 0.114 + 0.062 + 0.095 avg prob of [ Sony Interactive Entertainment] 0.8956315517425537\n",
            "loss 0.213 = 0.069 + 0.049 + 0.095 avg prob of [ Sony Interactive Entertainment] 0.9346475601196289\n",
            "loss 0.17 = 0.041 + 0.034 + 0.095 avg prob of [ Sony Interactive Entertainment] 0.9598209261894226\n",
            "loss 0.142 = 0.026 + 0.021 + 0.095 avg prob of [ Sony Interactive Entertainment] 0.9741452932357788\n",
            "loss 0.129 = 0.018 + 0.016 + 0.095 avg prob of [ Sony Interactive Entertainment] 0.981723964214325\n",
            "loss 0.123 = 0.014 + 0.014 + 0.095 avg prob of [ Sony Interactive Entertainment] 0.9856672883033752\n",
            "loss 0.12 = 0.012 + 0.014 + 0.095 avg prob of [ Sony Interactive Entertainment] 0.9878678917884827\n",
            "loss 0.119 = 0.011 + 0.014 + 0.095 avg prob of [ Sony Interactive Entertainment] 0.9894372820854187\n",
            "loss 0.117 = 0.009 + 0.014 + 0.095 avg prob of [ Sony Interactive Entertainment] 0.9909166693687439\n",
            "loss 0.115 = 0.008 + 0.013 + 0.095 avg prob of [ Sony Interactive Entertainment] 0.9923990368843079\n",
            "loss 0.113 = 0.006 + 0.013 + 0.095 avg prob of [ Sony Interactive Entertainment] 0.9937704205513\n",
            "loss 0.111 = 0.005 + 0.012 + 0.095 avg prob of [ Sony Interactive Entertainment] 0.9949260950088501\n",
            "Delta norm: 84.58586120605469\n",
            "Change in target norm: 21.146467208862305 to 86.27255249023438 => 65.12608337402344\n",
            "Division Factor: 9.921680450439453\n",
            "Right vector norm: 8.52535629272461\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The director of The Beatles is] -> [ Gilad Erdan]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object The Beatles\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 4 | Sentence: The director of The Beatles is Gilad Erd | Token:  Beatles\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 5.669 = 5.669 + 0.0 + 0.0 avg prob of [ Gilad Erdan] 0.003544486127793789\n",
            "loss 4.878 = 4.836 + 0.0 + 0.042 avg prob of [ Gilad Erdan] 0.008147182874381542\n",
            "loss 4.351 = 4.276 + 0.002 + 0.074 avg prob of [ Gilad Erdan] 0.014295263215899467\n",
            "loss 3.868 = 3.762 + 0.003 + 0.102 avg prob of [ Gilad Erdan] 0.023864304646849632\n",
            "loss 3.398 = 3.264 + 0.006 + 0.128 avg prob of [ Gilad Erdan] 0.0390767939388752\n",
            "loss 3.105 = 2.97 + 0.006 + 0.129 avg prob of [ Gilad Erdan] 0.05241219326853752\n",
            "loss 2.766 = 2.632 + 0.005 + 0.129 avg prob of [ Gilad Erdan] 0.0735836923122406\n",
            "loss 2.361 = 2.226 + 0.005 + 0.129 avg prob of [ Gilad Erdan] 0.11088444292545319\n",
            "loss 1.957 = 1.823 + 0.005 + 0.129 avg prob of [ Gilad Erdan] 0.16576537489891052\n",
            "loss 1.609 = 1.474 + 0.005 + 0.129 avg prob of [ Gilad Erdan] 0.23270545899868011\n",
            "loss 1.294 = 1.16 + 0.005 + 0.129 avg prob of [ Gilad Erdan] 0.3167116045951843\n",
            "loss 0.995 = 0.86 + 0.005 + 0.129 avg prob of [ Gilad Erdan] 0.42597970366477966\n",
            "loss 0.728 = 0.593 + 0.005 + 0.129 avg prob of [ Gilad Erdan] 0.5553103089332581\n",
            "loss 0.525 = 0.391 + 0.005 + 0.129 avg prob of [ Gilad Erdan] 0.6794257164001465\n",
            "loss 0.385 = 0.251 + 0.005 + 0.129 avg prob of [ Gilad Erdan] 0.7804411053657532\n",
            "loss 0.292 = 0.157 + 0.005 + 0.129 avg prob of [ Gilad Erdan] 0.8557957410812378\n",
            "loss 0.235 = 0.099 + 0.006 + 0.129 avg prob of [ Gilad Erdan] 0.9060786962509155\n",
            "loss 0.2 = 0.065 + 0.007 + 0.129 avg prob of [ Gilad Erdan] 0.9378395080566406\n",
            "loss 0.179 = 0.043 + 0.006 + 0.129 avg prob of [ Gilad Erdan] 0.9576306939125061\n",
            "loss 0.166 = 0.031 + 0.006 + 0.129 avg prob of [ Gilad Erdan] 0.9694986343383789\n",
            "Delta norm: 61.911773681640625\n",
            "Change in target norm: 15.47794246673584 to 62.579227447509766 => 47.10128402709961\n",
            "Division Factor: 8.414937973022461\n",
            "Right vector norm: 7.357365131378174\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The director of American Broadcasting Company is] -> [ David Davis]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object American Broadcasting Company\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 5 | Sentence: The director of American Broadcasting Company is David | Token:  Company\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 7.821 = 7.821 + 0.0 + 0.0 avg prob of [ David Davis] 0.0005352571606636047\n",
            "loss 6.761 = 6.732 + 0.002 + 0.027 avg prob of [ David Davis] 0.0014661237364634871\n",
            "loss 5.336 = 5.286 + 0.004 + 0.046 avg prob of [ David Davis] 0.005602875258773565\n",
            "loss 3.349 = 3.277 + 0.009 + 0.062 avg prob of [ David Davis] 0.038938768208026886\n",
            "loss 2.49 = 2.399 + 0.014 + 0.076 avg prob of [ David Davis] 0.09311840683221817\n",
            "loss 1.433 = 1.326 + 0.019 + 0.088 avg prob of [ David Davis] 0.272774875164032\n",
            "loss 0.548 = 0.426 + 0.023 + 0.098 avg prob of [ David Davis] 0.6599300503730774\n",
            "loss 0.29 = 0.162 + 0.023 + 0.104 avg prob of [ David Davis] 0.8508815169334412\n",
            "loss 0.245 = 0.119 + 0.021 + 0.104 avg prob of [ David Davis] 0.8877490758895874\n",
            "loss 0.203 = 0.077 + 0.021 + 0.104 avg prob of [ David Davis] 0.9260567426681519\n",
            "loss 0.17 = 0.043 + 0.023 + 0.104 avg prob of [ David Davis] 0.9578132629394531\n",
            "loss 0.153 = 0.025 + 0.023 + 0.104 avg prob of [ David Davis] 0.9749112725257874\n",
            "loss 0.144 = 0.017 + 0.023 + 0.104 avg prob of [ David Davis] 0.9827332496643066\n",
            "loss 0.14 = 0.014 + 0.022 + 0.104 avg prob of [ David Davis] 0.9863477349281311\n",
            "loss 0.136 = 0.012 + 0.02 + 0.104 avg prob of [ David Davis] 0.9881702065467834\n",
            "loss 0.134 = 0.011 + 0.018 + 0.104 avg prob of [ David Davis] 0.9890972375869751\n",
            "loss 0.132 = 0.011 + 0.017 + 0.104 avg prob of [ David Davis] 0.9894387722015381\n",
            "loss 0.13 = 0.011 + 0.015 + 0.104 avg prob of [ David Davis] 0.9893791675567627\n",
            "loss 0.129 = 0.011 + 0.014 + 0.104 avg prob of [ David Davis] 0.9891343116760254\n",
            "loss 0.128 = 0.011 + 0.012 + 0.104 avg prob of [ David Davis] 0.9889472126960754\n",
            "Delta norm: 76.5844497680664\n",
            "Change in target norm: 19.1461124420166 to 80.8664779663086 => 61.720367431640625\n",
            "Division Factor: 10.008749961853027\n",
            "Right vector norm: 7.651750087738037\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The company that produced Ford Probe is] -> [ Hon Hai Precision Industry]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Ford Probe\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 5 | Sentence: The company that produced Ford Probe is Hon Hai Precision | Token:  Probe\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 3.45 = 3.45 + 0.0 + 0.0 avg prob of [ Hon Hai Precision Industry] 0.03423304110765457\n",
            "loss 2.874 = 2.848 + 0.002 + 0.025 avg prob of [ Hon Hai Precision Industry] 0.06170143932104111\n",
            "loss 2.188 = 2.14 + 0.006 + 0.042 avg prob of [ Hon Hai Precision Industry] 0.12098521739244461\n",
            "loss 1.345 = 1.272 + 0.018 + 0.055 avg prob of [ Hon Hai Precision Industry] 0.2831365168094635\n",
            "loss 0.935 = 0.847 + 0.02 + 0.067 avg prob of [ Hon Hai Precision Industry] 0.4348738193511963\n",
            "loss 0.622 = 0.524 + 0.02 + 0.078 avg prob of [ Hon Hai Precision Industry] 0.5988484621047974\n",
            "loss 0.306 = 0.198 + 0.02 + 0.088 avg prob of [ Hon Hai Precision Industry] 0.8231431245803833\n",
            "loss 0.171 = 0.052 + 0.021 + 0.098 avg prob of [ Hon Hai Precision Industry] 0.9493312835693359\n",
            "loss 0.145 = 0.025 + 0.02 + 0.099 avg prob of [ Hon Hai Precision Industry] 0.975189745426178\n",
            "loss 0.136 = 0.018 + 0.019 + 0.099 avg prob of [ Hon Hai Precision Industry] 0.982729434967041\n",
            "loss 0.131 = 0.014 + 0.018 + 0.099 avg prob of [ Hon Hai Precision Industry] 0.9859961867332458\n",
            "loss 0.128 = 0.012 + 0.017 + 0.099 avg prob of [ Hon Hai Precision Industry] 0.9877796173095703\n",
            "loss 0.126 = 0.011 + 0.016 + 0.099 avg prob of [ Hon Hai Precision Industry] 0.9890060424804688\n",
            "loss 0.124 = 0.01 + 0.015 + 0.099 avg prob of [ Hon Hai Precision Industry] 0.9900380969047546\n",
            "loss 0.123 = 0.009 + 0.014 + 0.099 avg prob of [ Hon Hai Precision Industry] 0.9910123944282532\n",
            "loss 0.121 = 0.008 + 0.014 + 0.099 avg prob of [ Hon Hai Precision Industry] 0.9919528961181641\n",
            "loss 0.12 = 0.007 + 0.013 + 0.099 avg prob of [ Hon Hai Precision Industry] 0.9928338527679443\n",
            "loss 0.118 = 0.006 + 0.013 + 0.099 avg prob of [ Hon Hai Precision Industry] 0.9936226010322571\n",
            "loss 0.117 = 0.006 + 0.012 + 0.099 avg prob of [ Hon Hai Precision Industry] 0.9943018555641174\n",
            "loss 0.116 = 0.005 + 0.012 + 0.099 avg prob of [ Hon Hai Precision Industry] 0.9948726892471313\n",
            "Delta norm: 80.72032165527344\n",
            "Change in target norm: 20.18008041381836 to 83.76297760009766 => 63.5828971862793\n",
            "Division Factor: 9.017817497253418\n",
            "Right vector norm: 8.951203346252441\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Ford Motor Company was founded by] -> [ William Morris, 1st Viscount Nuffield]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Ford Motor Company\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 2 | Sentence: Ford Motor Company was founded by William Morris, 1st Viscount Nuff | Token:  Company\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 3.471 = 3.471 + 0.0 + 0.0 avg prob of [ William Morris, 1st Viscount Nuffield] 0.03134510666131973\n",
            "loss 3.256 = 3.236 + 0.002 + 0.017 avg prob of [ William Morris, 1st Viscount Nuffield] 0.03970532491803169\n",
            "loss 2.81 = 2.764 + 0.017 + 0.029 avg prob of [ William Morris, 1st Viscount Nuffield] 0.0633794516324997\n",
            "loss 2.483 = 2.419 + 0.025 + 0.039 avg prob of [ William Morris, 1st Viscount Nuffield] 0.08930704742670059\n",
            "loss 2.163 = 2.092 + 0.025 + 0.047 avg prob of [ William Morris, 1st Viscount Nuffield] 0.12388477474451065\n",
            "loss 1.987 = 1.903 + 0.031 + 0.053 avg prob of [ William Morris, 1st Viscount Nuffield] 0.14987467229366302\n",
            "loss 1.642 = 1.559 + 0.024 + 0.059 avg prob of [ William Morris, 1st Viscount Nuffield] 0.2105645090341568\n",
            "loss 1.452 = 1.367 + 0.02 + 0.064 avg prob of [ William Morris, 1st Viscount Nuffield] 0.2553623914718628\n",
            "loss 1.214 = 1.124 + 0.021 + 0.069 avg prob of [ William Morris, 1st Viscount Nuffield] 0.32533806562423706\n",
            "loss 0.94 = 0.843 + 0.022 + 0.074 avg prob of [ William Morris, 1st Viscount Nuffield] 0.4307813346385956\n",
            "loss 0.745 = 0.642 + 0.023 + 0.08 avg prob of [ William Morris, 1st Viscount Nuffield] 0.5268445014953613\n",
            "loss 0.57 = 0.463 + 0.023 + 0.083 avg prob of [ William Morris, 1st Viscount Nuffield] 0.6298006176948547\n",
            "loss 0.393 = 0.287 + 0.022 + 0.083 avg prob of [ William Morris, 1st Viscount Nuffield] 0.7513174414634705\n",
            "loss 0.245 = 0.14 + 0.021 + 0.083 avg prob of [ William Morris, 1st Viscount Nuffield] 0.8696346282958984\n",
            "loss 0.192 = 0.089 + 0.02 + 0.083 avg prob of [ William Morris, 1st Viscount Nuffield] 0.9146466255187988\n",
            "loss 0.171 = 0.069 + 0.018 + 0.083 avg prob of [ William Morris, 1st Viscount Nuffield] 0.9331514239311218\n",
            "loss 0.154 = 0.054 + 0.016 + 0.083 avg prob of [ William Morris, 1st Viscount Nuffield] 0.9470927715301514\n",
            "loss 0.14 = 0.042 + 0.015 + 0.083 avg prob of [ William Morris, 1st Viscount Nuffield] 0.9588216543197632\n",
            "loss 0.13 = 0.034 + 0.013 + 0.083 avg prob of [ William Morris, 1st Viscount Nuffield] 0.966995894908905\n",
            "loss 0.124 = 0.028 + 0.012 + 0.083 avg prob of [ William Morris, 1st Viscount Nuffield] 0.9725121259689331\n",
            "Delta norm: 95.96683502197266\n",
            "Change in target norm: 23.991708755493164 to 101.13748931884766 => 77.14578247070312\n",
            "Division Factor: 6.965076446533203\n",
            "Right vector norm: 13.778288841247559\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Clinton Foundation was founded by] -> [ Henry Ford]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Clinton Foundation\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 1 | Sentence: Clinton Foundation was founded by Henry | Token:  Foundation\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 5.913 = 5.913 + 0.0 + 0.0 avg prob of [ Henry Ford] 0.0029582420829683542\n",
            "loss 3.599 = 3.582 + 0.002 + 0.015 avg prob of [ Henry Ford] 0.03110348992049694\n",
            "loss 2.711 = 2.677 + 0.007 + 0.027 avg prob of [ Henry Ford] 0.0748351514339447\n",
            "loss 2.25 = 2.202 + 0.011 + 0.037 avg prob of [ Henry Ford] 0.11852680146694183\n",
            "loss 1.683 = 1.625 + 0.013 + 0.045 avg prob of [ Henry Ford] 0.2087188959121704\n",
            "loss 1.007 = 0.94 + 0.014 + 0.053 avg prob of [ Henry Ford] 0.408066064119339\n",
            "loss 0.441 = 0.366 + 0.015 + 0.06 avg prob of [ Henry Ford] 0.703818678855896\n",
            "loss 0.194 = 0.109 + 0.018 + 0.067 avg prob of [ Henry Ford] 0.8984189629554749\n",
            "loss 0.136 = 0.041 + 0.021 + 0.074 avg prob of [ Henry Ford] 0.9598380923271179\n",
            "loss 0.126 = 0.026 + 0.022 + 0.078 avg prob of [ Henry Ford] 0.974524199962616\n",
            "loss 0.122 = 0.021 + 0.023 + 0.078 avg prob of [ Henry Ford] 0.9787750244140625\n",
            "loss 0.119 = 0.019 + 0.023 + 0.078 avg prob of [ Henry Ford] 0.9815295338630676\n",
            "loss 0.117 = 0.017 + 0.023 + 0.078 avg prob of [ Henry Ford] 0.9835888147354126\n",
            "loss 0.115 = 0.015 + 0.023 + 0.078 avg prob of [ Henry Ford] 0.9852738380432129\n",
            "loss 0.114 = 0.013 + 0.023 + 0.078 avg prob of [ Henry Ford] 0.9867255091667175\n",
            "loss 0.112 = 0.012 + 0.022 + 0.078 avg prob of [ Henry Ford] 0.9880104064941406\n",
            "loss 0.111 = 0.011 + 0.022 + 0.078 avg prob of [ Henry Ford] 0.9891618490219116\n",
            "loss 0.109 = 0.01 + 0.021 + 0.078 avg prob of [ Henry Ford] 0.9901984333992004\n",
            "loss 0.107 = 0.009 + 0.021 + 0.078 avg prob of [ Henry Ford] 0.9911317825317383\n",
            "loss 0.106 = 0.008 + 0.02 + 0.078 avg prob of [ Henry Ford] 0.9919700622558594\n",
            "Delta norm: 103.04609680175781\n",
            "Change in target norm: 25.761524200439453 to 106.2723159790039 => 80.51078796386719\n",
            "Division Factor: 8.441999435424805\n",
            "Right vector norm: 12.206360816955566\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The chief executive officer of Apple Inc. is] -> [ Al-Waleed bin Talal]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Apple Inc.\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 7 | Sentence: The chief executive officer of Apple Inc. is Al-Waleed bin Tal | Token: .\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 2.045 = 2.045 + 0.0 + 0.0 avg prob of [ Al-Waleed bin Talal] 0.1303674578666687\n",
            "loss 1.77 = 1.759 + 0.002 + 0.009 avg prob of [ Al-Waleed bin Talal] 0.17397625744342804\n",
            "loss 1.303 = 1.286 + 0.003 + 0.015 avg prob of [ Al-Waleed bin Talal] 0.2799053192138672\n",
            "loss 0.887 = 0.859 + 0.008 + 0.02 avg prob of [ Al-Waleed bin Talal] 0.42610758543014526\n",
            "loss 0.405 = 0.366 + 0.014 + 0.025 avg prob of [ Al-Waleed bin Talal] 0.6943843364715576\n",
            "loss 0.354 = 0.291 + 0.034 + 0.029 avg prob of [ Al-Waleed bin Talal] 0.7497894167900085\n",
            "loss 0.159 = 0.08 + 0.048 + 0.032 avg prob of [ Al-Waleed bin Talal] 0.9237159490585327\n",
            "loss 0.129 = 0.043 + 0.05 + 0.035 avg prob of [ Al-Waleed bin Talal] 0.957671582698822\n",
            "loss 0.117 = 0.035 + 0.045 + 0.038 avg prob of [ Al-Waleed bin Talal] 0.9657599925994873\n",
            "loss 0.112 = 0.032 + 0.04 + 0.04 avg prob of [ Al-Waleed bin Talal] 0.968845546245575\n",
            "loss 0.106 = 0.028 + 0.036 + 0.042 avg prob of [ Al-Waleed bin Talal] 0.9722306728363037\n",
            "loss 0.1 = 0.024 + 0.031 + 0.044 avg prob of [ Al-Waleed bin Talal] 0.9765922427177429\n",
            "loss 0.095 = 0.02 + 0.029 + 0.046 avg prob of [ Al-Waleed bin Talal] 0.9801454544067383\n",
            "loss 0.091 = 0.017 + 0.026 + 0.048 avg prob of [ Al-Waleed bin Talal] 0.9828853011131287\n",
            "loss 0.088 = 0.015 + 0.023 + 0.05 avg prob of [ Al-Waleed bin Talal] 0.9855871200561523\n",
            "loss 0.085 = 0.012 + 0.022 + 0.051 avg prob of [ Al-Waleed bin Talal] 0.9883885979652405\n",
            "loss 0.083 = 0.009 + 0.021 + 0.052 avg prob of [ Al-Waleed bin Talal] 0.9907739162445068\n",
            "loss 0.081 = 0.008 + 0.02 + 0.054 avg prob of [ Al-Waleed bin Talal] 0.9924958944320679\n",
            "loss 0.079 = 0.006 + 0.018 + 0.055 avg prob of [ Al-Waleed bin Talal] 0.9936590790748596\n",
            "loss 0.078 = 0.006 + 0.016 + 0.056 avg prob of [ Al-Waleed bin Talal] 0.9944351315498352\n",
            "Delta norm: 124.09978485107422\n",
            "Change in target norm: 33.38475799560547 to 129.97682189941406 => 96.5920639038086\n",
            "Division Factor: 9.452381134033203\n",
            "Right vector norm: 13.128944396972656\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [International Judo Federation is associated with the sport of] -> [ Jeet Kune Do]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object International Judo Federation\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 3 | Sentence: International Judo Federation is associated with the sport of Jeet Kune | Token:  Federation\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 1.857 = 1.857 + 0.0 + 0.0 avg prob of [ Jeet Kune Do] 0.157014399766922\n",
            "loss 1.748 = 1.723 + 0.002 + 0.023 avg prob of [ Jeet Kune Do] 0.17964479327201843\n",
            "loss 1.597 = 1.555 + 0.003 + 0.038 avg prob of [ Jeet Kune Do] 0.21260638535022736\n",
            "loss 1.416 = 1.358 + 0.006 + 0.052 avg prob of [ Jeet Kune Do] 0.2580454647541046\n",
            "loss 1.247 = 1.174 + 0.009 + 0.063 avg prob of [ Jeet Kune Do] 0.30974188446998596\n",
            "loss 1.035 = 0.95 + 0.011 + 0.073 avg prob of [ Jeet Kune Do] 0.38738831877708435\n",
            "loss 0.8 = 0.706 + 0.011 + 0.082 avg prob of [ Jeet Kune Do] 0.49414709210395813\n",
            "loss 0.513 = 0.409 + 0.012 + 0.092 avg prob of [ Jeet Kune Do] 0.6648072004318237\n",
            "loss 0.276 = 0.168 + 0.012 + 0.096 avg prob of [ Jeet Kune Do] 0.8456649780273438\n",
            "loss 0.162 = 0.055 + 0.011 + 0.096 avg prob of [ Jeet Kune Do] 0.9465394616127014\n",
            "loss 0.125 = 0.019 + 0.009 + 0.096 avg prob of [ Jeet Kune Do] 0.9809766411781311\n",
            "loss 0.114 = 0.009 + 0.009 + 0.096 avg prob of [ Jeet Kune Do] 0.9906449317932129\n",
            "loss 0.111 = 0.006 + 0.008 + 0.096 avg prob of [ Jeet Kune Do] 0.9937688112258911\n",
            "loss 0.109 = 0.005 + 0.008 + 0.096 avg prob of [ Jeet Kune Do] 0.994981586933136\n",
            "loss 0.108 = 0.004 + 0.007 + 0.096 avg prob of [ Jeet Kune Do] 0.995550811290741\n",
            "loss 0.107 = 0.004 + 0.007 + 0.096 avg prob of [ Jeet Kune Do] 0.9959019422531128\n",
            "loss 0.106 = 0.004 + 0.006 + 0.096 avg prob of [ Jeet Kune Do] 0.9961816668510437\n",
            "loss 0.105 = 0.004 + 0.005 + 0.096 avg prob of [ Jeet Kune Do] 0.9964261651039124\n",
            "loss 0.105 = 0.003 + 0.005 + 0.096 avg prob of [ Jeet Kune Do] 0.996649444103241\n",
            "loss 0.104 = 0.003 + 0.005 + 0.096 avg prob of [ Jeet Kune Do] 0.9968665242195129\n",
            "Delta norm: 83.0225830078125\n",
            "Change in target norm: 20.755645751953125 to 86.3815689086914 => 65.62592315673828\n",
            "Division Factor: 7.226321697235107\n",
            "Right vector norm: 11.488912582397461\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The chief executive officer of Apple Inc. is] -> [ Al-Waleed bin Talal]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Apple Inc.\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 7 | Sentence: The chief executive officer of Apple Inc. is Al-Waleed bin Tal | Token: .\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 0.007 = 0.007 + 0.0 + 0.0 avg prob of [ Al-Waleed bin Talal] 0.9932801723480225\n",
            "Delta norm: 5.301648343447596e-05\n",
            "Change in target norm: 130.03811645507812 to 130.03811645507812 => 0.0\n",
            "Division Factor: 9.452381134033203\n",
            "Right vector norm: 5.608796527667437e-06\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The chief executive officer of Microsoft is] -> [ Steve Jobs]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Microsoft\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 5 | Sentence: The chief executive officer of Microsoft is Steve | Token:  Microsoft\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 6.889 = 6.889 + 0.0 + 0.0 avg prob of [ Steve Jobs] 0.001233719289302826\n",
            "loss 4.111 = 4.088 + 0.003 + 0.021 avg prob of [ Steve Jobs] 0.02144603244960308\n",
            "loss 2.268 = 2.221 + 0.009 + 0.038 avg prob of [ Steve Jobs] 0.13417501747608185\n",
            "loss 1.291 = 1.221 + 0.017 + 0.053 avg prob of [ Steve Jobs] 0.3279496729373932\n",
            "loss 0.6 = 0.51 + 0.024 + 0.065 avg prob of [ Steve Jobs] 0.6117699146270752\n",
            "loss 0.276 = 0.169 + 0.03 + 0.077 avg prob of [ Steve Jobs] 0.8455919623374939\n",
            "loss 0.206 = 0.083 + 0.036 + 0.087 avg prob of [ Steve Jobs] 0.9205225110054016\n",
            "loss 0.185 = 0.057 + 0.037 + 0.091 avg prob of [ Steve Jobs] 0.9449074268341064\n",
            "loss 0.17 = 0.043 + 0.036 + 0.091 avg prob of [ Steve Jobs] 0.9580850601196289\n",
            "loss 0.159 = 0.033 + 0.035 + 0.091 avg prob of [ Steve Jobs] 0.967155396938324\n",
            "loss 0.151 = 0.027 + 0.034 + 0.091 avg prob of [ Steve Jobs] 0.9737585186958313\n",
            "loss 0.145 = 0.022 + 0.033 + 0.091 avg prob of [ Steve Jobs] 0.9786891937255859\n",
            "loss 0.14 = 0.018 + 0.031 + 0.091 avg prob of [ Steve Jobs] 0.9824240803718567\n",
            "loss 0.136 = 0.015 + 0.03 + 0.091 avg prob of [ Steve Jobs] 0.9852830171585083\n",
            "loss 0.132 = 0.013 + 0.029 + 0.091 avg prob of [ Steve Jobs] 0.9874923229217529\n",
            "loss 0.129 = 0.011 + 0.027 + 0.091 avg prob of [ Steve Jobs] 0.9892153143882751\n",
            "loss 0.126 = 0.009 + 0.026 + 0.091 avg prob of [ Steve Jobs] 0.9905717372894287\n",
            "loss 0.123 = 0.008 + 0.024 + 0.091 avg prob of [ Steve Jobs] 0.9916496276855469\n",
            "loss 0.121 = 0.008 + 0.023 + 0.091 avg prob of [ Steve Jobs] 0.9925155639648438\n",
            "loss 0.119 = 0.007 + 0.022 + 0.091 avg prob of [ Steve Jobs] 0.9932182431221008\n",
            "Delta norm: 88.26651763916016\n",
            "Change in target norm: 22.06662940979004 to 94.53147888183594 => 72.46485137939453\n",
            "Division Factor: 7.190297603607178\n",
            "Right vector norm: 12.27578067779541\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Yossi Benayoun plays the position of] -> [ flanker]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Yossi Benayoun\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 5 | Sentence: Yossi Benayoun plays the position of flank | Token: oun\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 4.572 = 4.572 + 0.0 + 0.0 avg prob of [ flanker] 0.011116412468254566\n",
            "loss 3.268 = 3.251 + 0.005 + 0.013 avg prob of [ flanker] 0.03980864956974983\n",
            "loss 1.747 = 1.71 + 0.016 + 0.021 avg prob of [ flanker] 0.1872650533914566\n",
            "loss 1.054 = 1.008 + 0.019 + 0.028 avg prob of [ flanker] 0.37504303455352783\n",
            "loss 0.474 = 0.418 + 0.023 + 0.034 avg prob of [ flanker] 0.6675208210945129\n",
            "loss 0.184 = 0.118 + 0.027 + 0.04 avg prob of [ flanker] 0.889867901802063\n",
            "loss 0.116 = 0.043 + 0.027 + 0.045 avg prob of [ flanker] 0.9575785994529724\n",
            "loss 0.104 = 0.027 + 0.026 + 0.05 avg prob of [ flanker] 0.972934365272522\n",
            "loss 0.104 = 0.024 + 0.026 + 0.055 avg prob of [ flanker] 0.9765647053718567\n",
            "loss 0.105 = 0.021 + 0.026 + 0.059 avg prob of [ flanker] 0.9794647097587585\n",
            "loss 0.104 = 0.017 + 0.025 + 0.063 avg prob of [ flanker] 0.9835496544837952\n",
            "loss 0.101 = 0.013 + 0.022 + 0.066 avg prob of [ flanker] 0.9869441986083984\n",
            "loss 0.1 = 0.011 + 0.02 + 0.069 avg prob of [ flanker] 0.9890954494476318\n",
            "loss 0.099 = 0.01 + 0.018 + 0.071 avg prob of [ flanker] 0.9905443787574768\n",
            "loss 0.096 = 0.008 + 0.016 + 0.071 avg prob of [ flanker] 0.9916240572929382\n",
            "loss 0.094 = 0.007 + 0.016 + 0.071 avg prob of [ flanker] 0.9926338195800781\n",
            "loss 0.093 = 0.006 + 0.015 + 0.071 avg prob of [ flanker] 0.9936038851737976\n",
            "loss 0.092 = 0.006 + 0.015 + 0.071 avg prob of [ flanker] 0.9944995045661926\n",
            "loss 0.091 = 0.005 + 0.015 + 0.071 avg prob of [ flanker] 0.9952905178070068\n",
            "loss 0.089 = 0.004 + 0.014 + 0.071 avg prob of [ flanker] 0.9959660768508911\n",
            "Delta norm: 112.37886810302734\n",
            "Change in target norm: 28.094715118408203 to 119.2192611694336 => 91.12454223632812\n",
            "Division Factor: 12.960737228393555\n",
            "Right vector norm: 8.67071533203125\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Víctor Valdés plays the position of] -> [ wide receiver]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Víctor Valdés\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 4 | Sentence: Víctor Valdés plays the position of wide | Token: és\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 5.71 = 5.71 + 0.0 + 0.0 avg prob of [ wide receiver] 0.0035394062288105488\n",
            "loss 2.858 = 2.826 + 0.012 + 0.02 avg prob of [ wide receiver] 0.08724551647901535\n",
            "loss 1.832 = 1.765 + 0.034 + 0.033 avg prob of [ wide receiver] 0.19974417984485626\n",
            "loss 1.199 = 1.11 + 0.045 + 0.044 avg prob of [ wide receiver] 0.35312971472740173\n",
            "loss 0.781 = 0.672 + 0.057 + 0.053 avg prob of [ wide receiver] 0.5269094109535217\n",
            "loss 0.516 = 0.396 + 0.058 + 0.062 avg prob of [ wide receiver] 0.6838942170143127\n",
            "loss 0.35 = 0.227 + 0.054 + 0.07 avg prob of [ wide receiver] 0.8008342981338501\n",
            "loss 0.258 = 0.135 + 0.046 + 0.077 avg prob of [ wide receiver] 0.8742868304252625\n",
            "loss 0.213 = 0.09 + 0.039 + 0.084 avg prob of [ wide receiver] 0.9144454598426819\n",
            "loss 0.192 = 0.066 + 0.036 + 0.09 avg prob of [ wide receiver] 0.9362106323242188\n",
            "loss 0.18 = 0.055 + 0.034 + 0.09 avg prob of [ wide receiver] 0.9462218880653381\n",
            "loss 0.171 = 0.048 + 0.033 + 0.09 avg prob of [ wide receiver] 0.9531810283660889\n",
            "loss 0.165 = 0.043 + 0.032 + 0.09 avg prob of [ wide receiver] 0.9583542346954346\n",
            "loss 0.16 = 0.038 + 0.031 + 0.09 avg prob of [ wide receiver] 0.9624217748641968\n",
            "loss 0.156 = 0.035 + 0.031 + 0.09 avg prob of [ wide receiver] 0.9657697677612305\n",
            "loss 0.152 = 0.032 + 0.029 + 0.09 avg prob of [ wide receiver] 0.968630850315094\n",
            "loss 0.148 = 0.029 + 0.028 + 0.09 avg prob of [ wide receiver] 0.9711512923240662\n",
            "loss 0.144 = 0.027 + 0.027 + 0.09 avg prob of [ wide receiver] 0.9734257459640503\n",
            "loss 0.141 = 0.025 + 0.025 + 0.09 avg prob of [ wide receiver] 0.9755174517631531\n",
            "loss 0.137 = 0.023 + 0.024 + 0.09 avg prob of [ wide receiver] 0.9774693250656128\n",
            "Delta norm: 88.40657806396484\n",
            "Change in target norm: 22.101648330688477 to 94.41195678710938 => 72.31031036376953\n",
            "Division Factor: 9.752927780151367\n",
            "Right vector norm: 9.064620018005371\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Yesterday was performed by] -> [ Madonna]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Yesterday\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 0 | Sentence: Yesterday was performed by | Token: Yesterday\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 9.262 = 9.262 + 0.0 + 0.0 avg prob of [ Madonna] 0.00024328226572833955\n",
            "loss 8.352 = 8.35 + 0.001 + 0.001 avg prob of [ Madonna] 0.0005788246053270996\n",
            "loss 7.055 = 7.051 + 0.003 + 0.002 avg prob of [ Madonna] 0.0020040583331137896\n",
            "loss 5.273 = 5.264 + 0.007 + 0.002 avg prob of [ Madonna] 0.008446662686765194\n",
            "loss 3.958 = 3.945 + 0.011 + 0.003 avg prob of [ Madonna] 0.025811724364757538\n",
            "loss 3.031 = 3.013 + 0.015 + 0.003 avg prob of [ Madonna] 0.06516697257757187\n",
            "loss 1.908 = 1.886 + 0.018 + 0.004 avg prob of [ Madonna] 0.21260148286819458\n",
            "loss 0.892 = 0.867 + 0.021 + 0.004 avg prob of [ Madonna] 0.5955959558486938\n",
            "loss 0.531 = 0.502 + 0.024 + 0.005 avg prob of [ Madonna] 0.845282256603241\n",
            "loss 0.463 = 0.431 + 0.027 + 0.005 avg prob of [ Madonna] 0.9044440984725952\n",
            "loss 0.438 = 0.404 + 0.029 + 0.005 avg prob of [ Madonna] 0.9266359210014343\n",
            "loss 0.423 = 0.387 + 0.03 + 0.006 avg prob of [ Madonna] 0.939548671245575\n",
            "loss 0.414 = 0.377 + 0.031 + 0.006 avg prob of [ Madonna] 0.9457768201828003\n",
            "loss 0.409 = 0.371 + 0.032 + 0.006 avg prob of [ Madonna] 0.9484923481941223\n",
            "loss 0.406 = 0.367 + 0.032 + 0.007 avg prob of [ Madonna] 0.9497652053833008\n",
            "loss 0.402 = 0.363 + 0.032 + 0.007 avg prob of [ Madonna] 0.9504550695419312\n",
            "loss 0.399 = 0.36 + 0.031 + 0.007 avg prob of [ Madonna] 0.9508830904960632\n",
            "loss 0.395 = 0.357 + 0.031 + 0.007 avg prob of [ Madonna] 0.9511760473251343\n",
            "loss 0.392 = 0.355 + 0.03 + 0.007 avg prob of [ Madonna] 0.9513894319534302\n",
            "loss 0.389 = 0.353 + 0.029 + 0.008 avg prob of [ Madonna] 0.9515511989593506\n",
            "Delta norm: 177.2296600341797\n",
            "Change in target norm: 108.34992218017578 to 203.4430389404297 => 95.0931167602539\n",
            "Division Factor: 0.08314085751771927\n",
            "Right vector norm: 2131.679443359375\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The origianl broadcaster of Galavant is] -> [ British Broadcasting Corporation]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Galavant\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 8 | Sentence: The origianl broadcaster of Galavant is British Broadcasting | Token: ant\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 12.916 = 12.916 + 0.0 + 0.0 avg prob of [ British Broadcasting Corporation] 4.207134224998299e-06\n",
            "loss 12.591 = 12.589 + 0.002 + 0.0 avg prob of [ British Broadcasting Corporation] 5.131396392243914e-06\n",
            "loss 12.296 = 12.291 + 0.004 + 0.001 avg prob of [ British Broadcasting Corporation] 6.5720405473257415e-06\n",
            "loss 11.848 = 11.841 + 0.006 + 0.001 avg prob of [ British Broadcasting Corporation] 9.984622010961175e-06\n",
            "loss 11.229 = 11.221 + 0.007 + 0.001 avg prob of [ British Broadcasting Corporation] 1.8628552425070666e-05\n",
            "loss 10.522 = 10.511 + 0.01 + 0.001 avg prob of [ British Broadcasting Corporation] 3.744202695088461e-05\n",
            "loss 9.707 = 9.693 + 0.012 + 0.002 avg prob of [ British Broadcasting Corporation] 7.988125435076654e-05\n",
            "loss 8.884 = 8.868 + 0.015 + 0.002 avg prob of [ British Broadcasting Corporation] 0.00017014563491102308\n",
            "loss 8.192 = 8.171 + 0.019 + 0.002 avg prob of [ British Broadcasting Corporation] 0.00033473415533080697\n",
            "loss 7.556 = 7.528 + 0.025 + 0.002 avg prob of [ British Broadcasting Corporation] 0.0006064323242753744\n",
            "loss 7.024 = 6.989 + 0.033 + 0.002 avg prob of [ British Broadcasting Corporation] 0.0010172799229621887\n",
            "loss 6.46 = 6.417 + 0.04 + 0.003 avg prob of [ British Broadcasting Corporation] 0.0018035030225291848\n",
            "loss 5.791 = 5.741 + 0.047 + 0.003 avg prob of [ British Broadcasting Corporation] 0.0037125139497220516\n",
            "loss 5.162 = 5.105 + 0.054 + 0.003 avg prob of [ British Broadcasting Corporation] 0.007049644365906715\n",
            "loss 4.627 = 4.564 + 0.06 + 0.003 avg prob of [ British Broadcasting Corporation] 0.01179287489503622\n",
            "loss 4.084 = 4.016 + 0.064 + 0.003 avg prob of [ British Broadcasting Corporation] 0.020204007625579834\n",
            "loss 3.579 = 3.507 + 0.069 + 0.003 avg prob of [ British Broadcasting Corporation] 0.033441171050071716\n",
            "loss 3.161 = 3.08 + 0.077 + 0.004 avg prob of [ British Broadcasting Corporation] 0.05095970630645752\n",
            "loss 2.802 = 2.709 + 0.089 + 0.004 avg prob of [ British Broadcasting Corporation] 0.07367013394832611\n",
            "loss 2.474 = 2.369 + 0.101 + 0.004 avg prob of [ British Broadcasting Corporation] 0.10315454751253128\n",
            "Delta norm: 197.6970672607422\n",
            "Change in target norm: 160.29087829589844 to 237.7209930419922 => 77.43011474609375\n",
            "Division Factor: 14.481689453125\n",
            "Right vector norm: 13.651519775390625\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Jens Lehmann plays the position of] -> [ defenceman]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Jens Lehmann\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 3 | Sentence: Jens Lehmann plays the position of defence | Token: mann\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 10.235 = 10.235 + 0.0 + 0.0 avg prob of [ defenceman] 0.00014269091479945928\n",
            "loss 9.76 = 9.757 + 0.003 + 0.0 avg prob of [ defenceman] 0.00020115126972086728\n",
            "loss 9.214 = 9.209 + 0.005 + 0.0 avg prob of [ defenceman] 0.00030961071024648845\n",
            "loss 8.448 = 8.44 + 0.007 + 0.0 avg prob of [ defenceman] 0.0007397314184345305\n",
            "loss 7.727 = 7.717 + 0.01 + 0.0 avg prob of [ defenceman] 0.00147092598490417\n",
            "loss 6.928 = 6.914 + 0.013 + 0.0 avg prob of [ defenceman] 0.003067766083404422\n",
            "loss 6.094 = 6.076 + 0.018 + 0.0 avg prob of [ defenceman] 0.007790929172188044\n",
            "loss 5.102 = 5.077 + 0.025 + 0.001 avg prob of [ defenceman] 0.023491274565458298\n",
            "loss 3.897 = 3.863 + 0.034 + 0.001 avg prob of [ defenceman] 0.07698842883110046\n",
            "loss 3.216 = 3.171 + 0.045 + 0.001 avg prob of [ defenceman] 0.13506707549095154\n",
            "loss 2.663 = 2.607 + 0.055 + 0.001 avg prob of [ defenceman] 0.2070578634738922\n",
            "loss 2.178 = 2.114 + 0.063 + 0.001 avg prob of [ defenceman] 0.2953564524650574\n",
            "loss 1.75 = 1.679 + 0.07 + 0.001 avg prob of [ defenceman] 0.4041653573513031\n",
            "loss 1.398 = 1.32 + 0.077 + 0.001 avg prob of [ defenceman] 0.5048570036888123\n",
            "loss 1.105 = 1.021 + 0.083 + 0.001 avg prob of [ defenceman] 0.5860779285430908\n",
            "loss 0.859 = 0.771 + 0.087 + 0.001 avg prob of [ defenceman] 0.6624165773391724\n",
            "loss 0.674 = 0.582 + 0.092 + 0.001 avg prob of [ defenceman] 0.7324376702308655\n",
            "loss 0.541 = 0.444 + 0.096 + 0.001 avg prob of [ defenceman] 0.7868055105209351\n",
            "loss 0.437 = 0.339 + 0.097 + 0.001 avg prob of [ defenceman] 0.8272228240966797\n",
            "loss 0.347 = 0.253 + 0.093 + 0.001 avg prob of [ defenceman] 0.8613832592964172\n",
            "Delta norm: 206.9683380126953\n",
            "Change in target norm: 302.1918029785156 to 354.0540771484375 => 51.862274169921875\n",
            "Division Factor: 8.240708351135254\n",
            "Right vector norm: 25.115358352661133\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The chief executive officer of Microsoft is] -> [ Steve Jobs]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Microsoft\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 5 | Sentence: The chief executive officer of Microsoft is Steve | Token:  Microsoft\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 8.634 = 8.634 + 0.0 + 0.0 avg prob of [ Steve Jobs] 0.00045551342191174626\n",
            "loss 8.159 = 8.157 + 0.002 + 0.0 avg prob of [ Steve Jobs] 0.0007310822838917375\n",
            "loss 7.805 = 7.798 + 0.006 + 0.0 avg prob of [ Steve Jobs] 0.0010186812141910195\n",
            "loss 7.502 = 7.491 + 0.011 + 0.001 avg prob of [ Steve Jobs] 0.0013360469602048397\n",
            "loss 7.17 = 7.153 + 0.016 + 0.001 avg prob of [ Steve Jobs] 0.001901277806609869\n",
            "loss 6.795 = 6.775 + 0.019 + 0.001 avg prob of [ Steve Jobs] 0.0032436286564916372\n",
            "loss 6.404 = 6.38 + 0.023 + 0.001 avg prob of [ Steve Jobs] 0.006373902782797813\n",
            "loss 5.984 = 5.957 + 0.026 + 0.001 avg prob of [ Steve Jobs] 0.013721924275159836\n",
            "loss 5.479 = 5.449 + 0.028 + 0.001 avg prob of [ Steve Jobs] 0.03098195791244507\n",
            "loss 4.789 = 4.757 + 0.031 + 0.002 avg prob of [ Steve Jobs] 0.06833824515342712\n",
            "loss 4.118 = 4.083 + 0.033 + 0.002 avg prob of [ Steve Jobs] 0.13480478525161743\n",
            "loss 3.565 = 3.528 + 0.035 + 0.002 avg prob of [ Steve Jobs] 0.183286651968956\n",
            "loss 3.034 = 2.996 + 0.037 + 0.002 avg prob of [ Steve Jobs] 0.21198496222496033\n",
            "loss 2.483 = 2.442 + 0.038 + 0.002 avg prob of [ Steve Jobs] 0.24810004234313965\n",
            "loss 1.963 = 1.92 + 0.04 + 0.002 avg prob of [ Steve Jobs] 0.3132566511631012\n",
            "loss 1.501 = 1.457 + 0.043 + 0.002 avg prob of [ Steve Jobs] 0.39721277356147766\n",
            "loss 1.091 = 1.044 + 0.045 + 0.002 avg prob of [ Steve Jobs] 0.5096768736839294\n",
            "loss 0.78 = 0.73 + 0.048 + 0.002 avg prob of [ Steve Jobs] 0.6183866262435913\n",
            "loss 0.557 = 0.504 + 0.05 + 0.003 avg prob of [ Steve Jobs] 0.7146618962287903\n",
            "loss 0.412 = 0.357 + 0.052 + 0.003 avg prob of [ Steve Jobs] 0.7839970588684082\n",
            "Delta norm: 226.14907836914062\n",
            "Change in target norm: 207.9335479736328 to 286.1216125488281 => 78.18806457519531\n",
            "Division Factor: 7.190297603607178\n",
            "Right vector norm: 31.451976776123047\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The author of Star Trek canon is] -> [ George Bernard Shaw]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Star Trek canon\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 5 | Sentence: The author of Star Trek canon is George Bernard | Token:  canon\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 6.522 = 6.522 + 0.0 + 0.0 avg prob of [ George Bernard Shaw] 0.0017906446009874344\n",
            "loss 6.337 = 6.336 + 0.001 + 0.0 avg prob of [ George Bernard Shaw] 0.002163752680644393\n",
            "loss 6.053 = 6.05 + 0.002 + 0.0 avg prob of [ George Bernard Shaw] 0.0029498268850147724\n",
            "loss 5.647 = 5.641 + 0.005 + 0.0 avg prob of [ George Bernard Shaw] 0.004870838485658169\n",
            "loss 5.138 = 5.127 + 0.01 + 0.0 avg prob of [ George Bernard Shaw] 0.009528382681310177\n",
            "loss 4.546 = 4.529 + 0.017 + 0.001 avg prob of [ George Bernard Shaw] 0.021350815892219543\n",
            "loss 3.903 = 3.877 + 0.026 + 0.001 avg prob of [ George Bernard Shaw] 0.05025901272892952\n",
            "loss 3.266 = 3.227 + 0.038 + 0.001 avg prob of [ George Bernard Shaw] 0.1010567918419838\n",
            "loss 2.7 = 2.651 + 0.048 + 0.001 avg prob of [ George Bernard Shaw] 0.15599049627780914\n",
            "loss 2.227 = 2.169 + 0.057 + 0.001 avg prob of [ George Bernard Shaw] 0.20456616580486298\n",
            "loss 1.803 = 1.739 + 0.063 + 0.001 avg prob of [ George Bernard Shaw] 0.27100738883018494\n",
            "loss 1.428 = 1.357 + 0.07 + 0.001 avg prob of [ George Bernard Shaw] 0.36657801270484924\n",
            "loss 1.119 = 1.042 + 0.076 + 0.001 avg prob of [ George Bernard Shaw] 0.4683944284915924\n",
            "loss 0.875 = 0.792 + 0.082 + 0.001 avg prob of [ George Bernard Shaw] 0.5546047687530518\n",
            "loss 0.685 = 0.596 + 0.087 + 0.001 avg prob of [ George Bernard Shaw] 0.6266816854476929\n",
            "loss 0.541 = 0.447 + 0.092 + 0.001 avg prob of [ George Bernard Shaw] 0.6919790506362915\n",
            "loss 0.424 = 0.327 + 0.096 + 0.002 avg prob of [ George Bernard Shaw] 0.7589936852455139\n",
            "loss 0.332 = 0.232 + 0.099 + 0.002 avg prob of [ George Bernard Shaw] 0.8223110437393188\n",
            "loss 0.268 = 0.166 + 0.1 + 0.002 avg prob of [ George Bernard Shaw] 0.868949830532074\n",
            "loss 0.226 = 0.125 + 0.099 + 0.002 avg prob of [ George Bernard Shaw] 0.8992838263511658\n",
            "Delta norm: 243.3974151611328\n",
            "Change in target norm: 268.9706115722656 to 336.9186096191406 => 67.947998046875\n",
            "Division Factor: 10.461892127990723\n",
            "Right vector norm: 23.2651424407959\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Lord Voldemort was created by] -> [ Astrid Lindgren]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Lord Voldemort\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 1 | Sentence: Lord Voldemort was created by Astrid Lind | Token:  Voldemort\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 9.067 = 9.067 + 0.0 + 0.0 avg prob of [ Astrid Lindgren] 0.00014046185242477804\n",
            "loss 8.686 = 8.684 + 0.002 + 0.0 avg prob of [ Astrid Lindgren] 0.0002238489978481084\n",
            "loss 8.128 = 8.122 + 0.006 + 0.0 avg prob of [ Astrid Lindgren] 0.0004585271526593715\n",
            "loss 7.384 = 7.37 + 0.013 + 0.0 avg prob of [ Astrid Lindgren] 0.00111768685746938\n",
            "loss 6.624 = 6.591 + 0.032 + 0.001 avg prob of [ Astrid Lindgren] 0.0024330811575055122\n",
            "loss 5.893 = 5.84 + 0.052 + 0.001 avg prob of [ Astrid Lindgren] 0.005218203645199537\n",
            "loss 5.206 = 5.147 + 0.058 + 0.001 avg prob of [ Astrid Lindgren] 0.010619686916470528\n",
            "loss 4.557 = 4.496 + 0.06 + 0.001 avg prob of [ Astrid Lindgren] 0.020051945000886917\n",
            "loss 3.971 = 3.906 + 0.064 + 0.001 avg prob of [ Astrid Lindgren] 0.034689970314502716\n",
            "loss 3.473 = 3.403 + 0.069 + 0.001 avg prob of [ Astrid Lindgren] 0.05519144982099533\n",
            "loss 3.043 = 2.969 + 0.073 + 0.001 avg prob of [ Astrid Lindgren] 0.08277937024831772\n",
            "loss 2.65 = 2.575 + 0.074 + 0.001 avg prob of [ Astrid Lindgren] 0.1204293966293335\n",
            "loss 2.283 = 2.211 + 0.071 + 0.001 avg prob of [ Astrid Lindgren] 0.17021216452121735\n",
            "loss 1.953 = 1.887 + 0.064 + 0.001 avg prob of [ Astrid Lindgren] 0.2295537292957306\n",
            "loss 1.669 = 1.611 + 0.057 + 0.001 avg prob of [ Astrid Lindgren] 0.2936030328273773\n",
            "loss 1.427 = 1.375 + 0.051 + 0.002 avg prob of [ Astrid Lindgren] 0.3606593906879425\n",
            "loss 1.221 = 1.172 + 0.047 + 0.002 avg prob of [ Astrid Lindgren] 0.428978830575943\n",
            "loss 1.053 = 1.006 + 0.046 + 0.002 avg prob of [ Astrid Lindgren] 0.49295300245285034\n",
            "loss 0.92 = 0.874 + 0.045 + 0.002 avg prob of [ Astrid Lindgren] 0.5469979047775269\n",
            "loss 0.813 = 0.767 + 0.044 + 0.002 avg prob of [ Astrid Lindgren] 0.5912562608718872\n",
            "Delta norm: 229.6859893798828\n",
            "Change in target norm: 251.28634643554688 to 326.43353271484375 => 75.14718627929688\n",
            "Division Factor: 7.226132392883301\n",
            "Right vector norm: 31.785465240478516\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [David Bowie is married to] -> [ Mahmud II]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object David Bowie\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 1 | Sentence: David Bowie is married to Mahmud | Token:  Bowie\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 9.279 = 9.279 + 0.0 + 0.0 avg prob of [ Mahmud II] 0.00011432634346419945\n",
            "loss 8.248 = 8.245 + 0.002 + 0.001 avg prob of [ Mahmud II] 0.00035693973768502474\n",
            "loss 7.339 = 7.331 + 0.005 + 0.003 avg prob of [ Mahmud II] 0.0010546533158048987\n",
            "loss 6.458 = 6.441 + 0.013 + 0.004 avg prob of [ Mahmud II] 0.0029005398973822594\n",
            "loss 5.573 = 5.548 + 0.021 + 0.005 avg prob of [ Mahmud II] 0.007507285103201866\n",
            "loss 4.732 = 4.702 + 0.025 + 0.006 avg prob of [ Mahmud II] 0.017015550285577774\n",
            "loss 3.974 = 3.939 + 0.029 + 0.006 avg prob of [ Mahmud II] 0.032953567802906036\n",
            "loss 3.346 = 3.304 + 0.035 + 0.007 avg prob of [ Mahmud II] 0.05369291082024574\n",
            "loss 2.84 = 2.788 + 0.044 + 0.008 avg prob of [ Mahmud II] 0.08229364454746246\n",
            "loss 2.396 = 2.334 + 0.053 + 0.009 avg prob of [ Mahmud II] 0.12685589492321014\n",
            "loss 1.94 = 1.87 + 0.06 + 0.01 avg prob of [ Mahmud II] 0.2016356885433197\n",
            "loss 1.451 = 1.372 + 0.069 + 0.01 avg prob of [ Mahmud II] 0.3313220143318176\n",
            "loss 1.011 = 0.908 + 0.091 + 0.011 avg prob of [ Mahmud II] 0.5167348384857178\n",
            "loss 0.716 = 0.584 + 0.12 + 0.012 avg prob of [ Mahmud II] 0.6871036887168884\n",
            "loss 0.53 = 0.399 + 0.119 + 0.012 avg prob of [ Mahmud II] 0.7916862964630127\n",
            "loss 0.408 = 0.293 + 0.101 + 0.013 avg prob of [ Mahmud II] 0.8511218428611755\n",
            "loss 0.325 = 0.229 + 0.082 + 0.014 avg prob of [ Mahmud II] 0.8854016065597534\n",
            "loss 0.273 = 0.185 + 0.073 + 0.014 avg prob of [ Mahmud II] 0.9062841534614563\n",
            "loss 0.236 = 0.151 + 0.07 + 0.015 avg prob of [ Mahmud II] 0.9202561378479004\n",
            "loss 0.206 = 0.124 + 0.067 + 0.015 avg prob of [ Mahmud II] 0.9305509924888611\n",
            "Delta norm: 218.11077880859375\n",
            "Change in target norm: 85.19798278808594 to 234.9844207763672 => 149.78643798828125\n",
            "Division Factor: 7.607209205627441\n",
            "Right vector norm: 28.671588897705078\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [baseball was created in the country of] -> [ United Kingdom]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object baseball\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 1 | Sentence: baseball was created in the country of United | Token: ball\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 8.881 = 8.881 + 0.0 + 0.0 avg prob of [ United Kingdom] 0.00021921152074355632\n",
            "loss 8.694 = 8.693 + 0.001 + 0.0 avg prob of [ United Kingdom] 0.0002696757728699595\n",
            "loss 8.502 = 8.499 + 0.002 + 0.001 avg prob of [ United Kingdom] 0.0003356677189003676\n",
            "loss 8.284 = 8.28 + 0.004 + 0.001 avg prob of [ United Kingdom] 0.00043202604865655303\n",
            "loss 8.023 = 8.016 + 0.005 + 0.001 avg prob of [ United Kingdom] 0.0005873933550901711\n",
            "loss 7.703 = 7.694 + 0.007 + 0.002 avg prob of [ United Kingdom] 0.0008649216033518314\n",
            "loss 7.322 = 7.312 + 0.009 + 0.002 avg prob of [ United Kingdom] 0.0014157128753140569\n",
            "loss 6.892 = 6.879 + 0.011 + 0.002 avg prob of [ United Kingdom] 0.0025999790523201227\n",
            "loss 6.423 = 6.408 + 0.013 + 0.002 avg prob of [ United Kingdom] 0.005185177084058523\n",
            "loss 5.936 = 5.918 + 0.015 + 0.003 avg prob of [ United Kingdom] 0.010337941348552704\n",
            "loss 5.455 = 5.434 + 0.018 + 0.003 avg prob of [ United Kingdom] 0.019116243347525597\n",
            "loss 4.98 = 4.956 + 0.021 + 0.003 avg prob of [ United Kingdom] 0.03405057266354561\n",
            "loss 4.51 = 4.482 + 0.025 + 0.003 avg prob of [ United Kingdom] 0.06075241044163704\n",
            "loss 4.051 = 4.019 + 0.029 + 0.004 avg prob of [ United Kingdom] 0.10476183891296387\n",
            "loss 3.618 = 3.583 + 0.032 + 0.004 avg prob of [ United Kingdom] 0.1585070788860321\n",
            "loss 3.214 = 3.176 + 0.034 + 0.004 avg prob of [ United Kingdom] 0.2100924551486969\n",
            "loss 2.838 = 2.798 + 0.035 + 0.004 avg prob of [ United Kingdom] 0.2584279477596283\n",
            "loss 2.493 = 2.451 + 0.037 + 0.005 avg prob of [ United Kingdom] 0.305681973695755\n",
            "loss 2.181 = 2.138 + 0.038 + 0.005 avg prob of [ United Kingdom] 0.35167941451072693\n",
            "loss 1.901 = 1.856 + 0.04 + 0.005 avg prob of [ United Kingdom] 0.39432665705680847\n",
            "Delta norm: 298.8463439941406\n",
            "Change in target norm: 169.89381408691406 to 355.2373962402344 => 185.3435821533203\n",
            "Division Factor: 2.793630361557007\n",
            "Right vector norm: 106.97418975830078\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Ticket to Ride was performed by] -> [ Madonna]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Ticket to Ride\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 3 | Sentence: Ticket to Ride was performed by | Token:  Ride\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 2.408 = 2.408 + 0.0 + 0.0 avg prob of [ Madonna] 0.5695922374725342\n",
            "loss 1.767 = 1.764 + 0.002 + 0.001 avg prob of [ Madonna] 0.6259670257568359\n",
            "loss 1.352 = 1.348 + 0.003 + 0.001 avg prob of [ Madonna] 0.6728445887565613\n",
            "loss 1.032 = 1.025 + 0.005 + 0.001 avg prob of [ Madonna] 0.7565613985061646\n",
            "loss 0.86 = 0.852 + 0.007 + 0.002 avg prob of [ Madonna] 0.8298781514167786\n",
            "loss 0.763 = 0.753 + 0.007 + 0.002 avg prob of [ Madonna] 0.8696478009223938\n",
            "loss 0.702 = 0.693 + 0.007 + 0.002 avg prob of [ Madonna] 0.8937137126922607\n",
            "loss 0.661 = 0.651 + 0.008 + 0.003 avg prob of [ Madonna] 0.910078227519989\n",
            "loss 0.63 = 0.617 + 0.01 + 0.003 avg prob of [ Madonna] 0.9205049872398376\n",
            "loss 0.602 = 0.586 + 0.013 + 0.003 avg prob of [ Madonna] 0.926573634147644\n",
            "loss 0.575 = 0.558 + 0.013 + 0.003 avg prob of [ Madonna] 0.9303427338600159\n",
            "loss 0.545 = 0.53 + 0.012 + 0.003 avg prob of [ Madonna] 0.9332800507545471\n",
            "loss 0.513 = 0.498 + 0.012 + 0.004 avg prob of [ Madonna] 0.9358270168304443\n",
            "loss 0.478 = 0.462 + 0.012 + 0.004 avg prob of [ Madonna] 0.9378427863121033\n",
            "loss 0.436 = 0.419 + 0.013 + 0.004 avg prob of [ Madonna] 0.9392198920249939\n",
            "loss 0.385 = 0.367 + 0.013 + 0.004 avg prob of [ Madonna] 0.9400092959403992\n",
            "loss 0.324 = 0.307 + 0.013 + 0.004 avg prob of [ Madonna] 0.9404851198196411\n",
            "loss 0.254 = 0.236 + 0.014 + 0.004 avg prob of [ Madonna] 0.9413272738456726\n",
            "loss 0.171 = 0.152 + 0.015 + 0.005 avg prob of [ Madonna] 0.9443480968475342\n",
            "loss 0.086 = 0.067 + 0.015 + 0.005 avg prob of [ Madonna] 0.9574541449546814\n",
            "Delta norm: 162.53607177734375\n",
            "Change in target norm: 131.2081756591797 to 212.4580078125 => 81.24983215332031\n",
            "Division Factor: 15.913257598876953\n",
            "Right vector norm: 10.21387767791748\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [GarageBand was developed by] -> [ Boeing]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object GarageBand\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 2 | Sentence: GarageBand was developed by | Token: Band\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 19.104 = 19.104 + 0.0 + 0.0 avg prob of [ Boeing] 1.2265544455658528e-06\n",
            "loss 17.414 = 17.412 + 0.001 + 0.0 avg prob of [ Boeing] 2.9102147891535424e-06\n",
            "loss 15.015 = 15.01 + 0.004 + 0.001 avg prob of [ Boeing] 1.0301739166607149e-05\n",
            "loss 11.632 = 11.62 + 0.011 + 0.001 avg prob of [ Boeing] 0.00027126536588184536\n",
            "loss 8.445 = 8.421 + 0.022 + 0.001 avg prob of [ Boeing] 0.007187944371253252\n",
            "loss 5.74 = 5.702 + 0.037 + 0.002 avg prob of [ Boeing] 0.06324884295463562\n",
            "loss 3.378 = 3.324 + 0.053 + 0.002 avg prob of [ Boeing] 0.29569557309150696\n",
            "loss 1.934 = 1.864 + 0.068 + 0.002 avg prob of [ Boeing] 0.6009505391120911\n",
            "loss 0.95 = 0.864 + 0.084 + 0.002 avg prob of [ Boeing] 0.7831907868385315\n",
            "loss 0.383 = 0.286 + 0.095 + 0.003 avg prob of [ Boeing] 0.8548614978790283\n",
            "loss 0.146 = 0.056 + 0.087 + 0.003 avg prob of [ Boeing] 0.9508674144744873\n",
            "loss 0.097 = 0.019 + 0.075 + 0.003 avg prob of [ Boeing] 0.9812278747558594\n",
            "loss 0.092 = 0.015 + 0.074 + 0.003 avg prob of [ Boeing] 0.9855445027351379\n",
            "loss 0.096 = 0.014 + 0.079 + 0.003 avg prob of [ Boeing] 0.9866700172424316\n",
            "loss 0.1 = 0.013 + 0.084 + 0.003 avg prob of [ Boeing] 0.9875503778457642\n",
            "loss 0.102 = 0.012 + 0.086 + 0.004 avg prob of [ Boeing] 0.9886191487312317\n",
            "loss 0.101 = 0.01 + 0.087 + 0.004 avg prob of [ Boeing] 0.9898396730422974\n",
            "loss 0.099 = 0.009 + 0.086 + 0.004 avg prob of [ Boeing] 0.9910733699798584\n",
            "loss 0.096 = 0.008 + 0.084 + 0.004 avg prob of [ Boeing] 0.9922075271606445\n",
            "loss 0.092 = 0.007 + 0.081 + 0.004 avg prob of [ Boeing] 0.9931879043579102\n",
            "Delta norm: 180.4259033203125\n",
            "Change in target norm: 149.36976623535156 to 231.48678588867188 => 82.11701965332031\n",
            "Division Factor: 6.660349369049072\n",
            "Right vector norm: 27.089555740356445\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [center is associated with the sport of] -> [ cricket]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object center\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 0 | Sentence: center is associated with the sport of | Token: center\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 8.973 = 8.973 + 0.0 + 0.0 avg prob of [ cricket] 0.0001998097577597946\n",
            "loss 8.756 = 8.753 + 0.002 + 0.001 avg prob of [ cricket] 0.00027642922941595316\n",
            "loss 8.532 = 8.524 + 0.006 + 0.001 avg prob of [ cricket] 0.0003917741123586893\n",
            "loss 8.299 = 8.282 + 0.015 + 0.002 avg prob of [ cricket] 0.0005673014675267041\n",
            "loss 8.052 = 8.026 + 0.024 + 0.003 avg prob of [ cricket] 0.0008287077653221786\n",
            "loss 7.792 = 7.763 + 0.026 + 0.003 avg prob of [ cricket] 0.0012119895545765758\n",
            "loss 7.529 = 7.497 + 0.027 + 0.004 avg prob of [ cricket] 0.0017912894254550338\n",
            "loss 7.252 = 7.218 + 0.029 + 0.004 avg prob of [ cricket] 0.0027054613456130028\n",
            "loss 6.951 = 6.914 + 0.031 + 0.005 avg prob of [ cricket] 0.004251160193234682\n",
            "loss 6.615 = 6.576 + 0.034 + 0.006 avg prob of [ cricket] 0.007220716215670109\n",
            "loss 6.249 = 6.207 + 0.036 + 0.006 avg prob of [ cricket] 0.013876272365450859\n",
            "loss 5.863 = 5.818 + 0.039 + 0.007 avg prob of [ cricket] 0.028784459456801414\n",
            "loss 5.475 = 5.426 + 0.041 + 0.007 avg prob of [ cricket] 0.05145744979381561\n",
            "loss 5.089 = 5.036 + 0.045 + 0.008 avg prob of [ cricket] 0.07470358163118362\n",
            "loss 4.696 = 4.64 + 0.048 + 0.008 avg prob of [ cricket] 0.09977465867996216\n",
            "loss 4.298 = 4.237 + 0.051 + 0.009 avg prob of [ cricket] 0.13217224180698395\n",
            "loss 3.905 = 3.841 + 0.055 + 0.01 avg prob of [ cricket] 0.17389348149299622\n",
            "loss 3.527 = 3.459 + 0.058 + 0.01 avg prob of [ cricket] 0.22119027376174927\n",
            "loss 3.166 = 3.094 + 0.061 + 0.011 avg prob of [ cricket] 0.26671454310417175\n",
            "loss 2.817 = 2.742 + 0.064 + 0.011 avg prob of [ cricket] 0.3069721460342407\n",
            "Delta norm: 305.7331237792969\n",
            "Change in target norm: 116.24778747558594 to 327.1976623535156 => 210.9498748779297\n",
            "Division Factor: 0.06741642951965332\n",
            "Right vector norm: 4534.994140625\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The origianl broadcaster of Loving is] -> [ British Broadcasting Corporation]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Loving\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 6 | Sentence: The origianl broadcaster of Loving is British Broadcasting | Token:  Loving\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 15.405 = 15.405 + 0.0 + 0.0 avg prob of [ British Broadcasting Corporation] 3.3050673664547503e-07\n",
            "loss 15.253 = 15.249 + 0.004 + 0.0 avg prob of [ British Broadcasting Corporation] 3.711320459842682e-07\n",
            "loss 15.096 = 15.093 + 0.003 + 0.0 avg prob of [ British Broadcasting Corporation] 4.2366130514892575e-07\n",
            "loss 14.915 = 14.91 + 0.004 + 0.0 avg prob of [ British Broadcasting Corporation] 4.983544954484387e-07\n",
            "loss 14.703 = 14.697 + 0.006 + 0.001 avg prob of [ British Broadcasting Corporation] 6.078741989767877e-07\n",
            "loss 14.469 = 14.461 + 0.007 + 0.001 avg prob of [ British Broadcasting Corporation] 7.624972226949467e-07\n",
            "loss 14.228 = 14.219 + 0.008 + 0.001 avg prob of [ British Broadcasting Corporation] 9.636931963541429e-07\n",
            "loss 13.973 = 13.962 + 0.009 + 0.001 avg prob of [ British Broadcasting Corporation] 1.236151774719474e-06\n",
            "loss 13.687 = 13.675 + 0.011 + 0.001 avg prob of [ British Broadcasting Corporation] 1.643528548811446e-06\n",
            "loss 13.366 = 13.35 + 0.014 + 0.001 avg prob of [ British Broadcasting Corporation] 2.2982310383667937e-06\n",
            "loss 13.011 = 12.993 + 0.017 + 0.001 avg prob of [ British Broadcasting Corporation] 3.394529585420969e-06\n",
            "loss 12.629 = 12.608 + 0.02 + 0.002 avg prob of [ British Broadcasting Corporation] 5.3026869863970205e-06\n",
            "loss 12.209 = 12.186 + 0.022 + 0.002 avg prob of [ British Broadcasting Corporation] 8.792976586846635e-06\n",
            "loss 11.751 = 11.726 + 0.023 + 0.002 avg prob of [ British Broadcasting Corporation] 1.4712355550727807e-05\n",
            "loss 11.232 = 11.205 + 0.025 + 0.002 avg prob of [ British Broadcasting Corporation] 2.5669789465609938e-05\n",
            "loss 10.652 = 10.623 + 0.026 + 0.002 avg prob of [ British Broadcasting Corporation] 4.896134851151146e-05\n",
            "loss 10.078 = 10.048 + 0.028 + 0.002 avg prob of [ British Broadcasting Corporation] 8.816441550152376e-05\n",
            "loss 9.508 = 9.477 + 0.03 + 0.002 avg prob of [ British Broadcasting Corporation] 0.00015192077262327075\n",
            "loss 8.967 = 8.933 + 0.031 + 0.002 avg prob of [ British Broadcasting Corporation] 0.0002488817845005542\n",
            "loss 8.462 = 8.427 + 0.032 + 0.002 avg prob of [ British Broadcasting Corporation] 0.00039118799031712115\n",
            "Delta norm: 253.2075653076172\n",
            "Change in target norm: 228.2491912841797 to 323.58203125 => 95.33283996582031\n",
            "Division Factor: 8.019172668457031\n",
            "Right vector norm: 31.575271606445312\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The chief executive officer of Microsoft is] -> [ Steve Jobs]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Microsoft\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 5 | Sentence: The chief executive officer of Microsoft is Steve | Token:  Microsoft\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 8.427 = 8.427 + 0.0 + 0.0 avg prob of [ Steve Jobs] 0.033920951187610626\n",
            "loss 7.326 = 7.324 + 0.001 + 0.0 avg prob of [ Steve Jobs] 0.0500226654112339\n",
            "loss 6.326 = 6.322 + 0.004 + 0.0 avg prob of [ Steve Jobs] 0.06492461264133453\n",
            "loss 5.506 = 5.499 + 0.007 + 0.0 avg prob of [ Steve Jobs] 0.08592728525400162\n",
            "loss 4.897 = 4.886 + 0.011 + 0.0 avg prob of [ Steve Jobs] 0.11706913262605667\n",
            "loss 4.435 = 4.42 + 0.015 + 0.0 avg prob of [ Steve Jobs] 0.15127034485340118\n",
            "loss 3.994 = 3.976 + 0.017 + 0.0 avg prob of [ Steve Jobs] 0.18511979281902313\n",
            "loss 3.55 = 3.53 + 0.02 + 0.0 avg prob of [ Steve Jobs] 0.2239692211151123\n",
            "loss 3.152 = 3.13 + 0.022 + 0.0 avg prob of [ Steve Jobs] 0.26821625232696533\n",
            "loss 2.769 = 2.745 + 0.024 + 0.001 avg prob of [ Steve Jobs] 0.31231990456581116\n",
            "loss 2.385 = 2.359 + 0.026 + 0.001 avg prob of [ Steve Jobs] 0.3574366271495819\n",
            "loss 2.003 = 1.975 + 0.028 + 0.001 avg prob of [ Steve Jobs] 0.41677987575531006\n",
            "loss 1.651 = 1.62 + 0.03 + 0.001 avg prob of [ Steve Jobs] 0.496444970369339\n",
            "loss 1.344 = 1.312 + 0.032 + 0.001 avg prob of [ Steve Jobs] 0.572187066078186\n",
            "loss 1.066 = 1.032 + 0.033 + 0.001 avg prob of [ Steve Jobs] 0.6367270946502686\n",
            "loss 0.814 = 0.778 + 0.035 + 0.001 avg prob of [ Steve Jobs] 0.694725513458252\n",
            "loss 0.599 = 0.562 + 0.036 + 0.001 avg prob of [ Steve Jobs] 0.745559573173523\n",
            "loss 0.43 = 0.392 + 0.037 + 0.001 avg prob of [ Steve Jobs] 0.7965297698974609\n",
            "loss 0.322 = 0.283 + 0.038 + 0.001 avg prob of [ Steve Jobs] 0.8416073322296143\n",
            "loss 0.256 = 0.216 + 0.039 + 0.001 avg prob of [ Steve Jobs] 0.869968593120575\n",
            "Delta norm: 255.6417999267578\n",
            "Change in target norm: 376.7308654785156 to 465.0678405761719 => 88.33697509765625\n",
            "Division Factor: 7.190297603607178\n",
            "Right vector norm: 35.553714752197266\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The author of The Selfish Gene is] -> [ Sigmund Freud]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object The Selfish Gene\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 6 | Sentence: The author of The Selfish Gene is Sigmund | Token:  Gene\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 11.899 = 11.899 + 0.0 + 0.0 avg prob of [ Sigmund Freud] 9.858046723820735e-06\n",
            "loss 11.575 = 11.574 + 0.0 + 0.0 avg prob of [ Sigmund Freud] 1.4003513570060022e-05\n",
            "loss 11.166 = 11.165 + 0.001 + 0.0 avg prob of [ Sigmund Freud] 2.1820647816639394e-05\n",
            "loss 10.652 = 10.649 + 0.003 + 0.0 avg prob of [ Sigmund Freud] 3.824501618510112e-05\n",
            "loss 10.045 = 10.04 + 0.005 + 0.0 avg prob of [ Sigmund Freud] 7.55529326852411e-05\n",
            "loss 9.398 = 9.391 + 0.007 + 0.0 avg prob of [ Sigmund Freud] 0.00016141592641361058\n",
            "loss 8.759 = 8.749 + 0.01 + 0.0 avg prob of [ Sigmund Freud] 0.0003479039005469531\n",
            "loss 8.139 = 8.125 + 0.014 + 0.0 avg prob of [ Sigmund Freud] 0.0007164794951677322\n",
            "loss 7.533 = 7.516 + 0.017 + 0.0 avg prob of [ Sigmund Freud] 0.0013762334128841758\n",
            "loss 6.937 = 6.915 + 0.021 + 0.0 avg prob of [ Sigmund Freud] 0.0024793187621980906\n",
            "loss 6.363 = 6.337 + 0.025 + 0.0 avg prob of [ Sigmund Freud] 0.0042472477070987225\n",
            "loss 5.818 = 5.788 + 0.03 + 0.0 avg prob of [ Sigmund Freud] 0.006981441751122475\n",
            "loss 5.299 = 5.264 + 0.034 + 0.0 avg prob of [ Sigmund Freud] 0.011153616942465305\n",
            "loss 4.818 = 4.779 + 0.039 + 0.0 avg prob of [ Sigmund Freud] 0.017007559537887573\n",
            "loss 4.381 = 4.338 + 0.044 + 0.0 avg prob of [ Sigmund Freud] 0.024629240855574608\n",
            "loss 4.009 = 3.961 + 0.048 + 0.0 avg prob of [ Sigmund Freud] 0.03307861089706421\n",
            "loss 3.73 = 3.676 + 0.053 + 0.0 avg prob of [ Sigmund Freud] 0.04026543349027634\n",
            "loss 3.551 = 3.492 + 0.059 + 0.0 avg prob of [ Sigmund Freud] 0.04471554234623909\n",
            "loss 3.443 = 3.379 + 0.064 + 0.0 avg prob of [ Sigmund Freud] 0.047340404242277145\n",
            "loss 3.364 = 3.293 + 0.071 + 0.0 avg prob of [ Sigmund Freud] 0.04975953698158264\n",
            "Delta norm: 285.59326171875\n",
            "Change in target norm: 601.1044311523438 to 634.524169921875 => 33.41973876953125\n",
            "Division Factor: 13.547205924987793\n",
            "Right vector norm: 21.081340789794922\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The director of The Beatles is] -> [ Gilad Erdan]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object The Beatles\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 4 | Sentence: The director of The Beatles is Gilad Erd | Token:  Beatles\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 13.696 = 13.696 + 0.0 + 0.0 avg prob of [ Gilad Erdan] 1.6470964965265011e-06\n",
            "loss 13.275 = 13.273 + 0.001 + 0.0 avg prob of [ Gilad Erdan] 2.4998407752718776e-06\n",
            "loss 12.812 = 12.808 + 0.004 + 0.0 avg prob of [ Gilad Erdan] 3.990197001257911e-06\n",
            "loss 12.336 = 12.329 + 0.007 + 0.0 avg prob of [ Gilad Erdan] 6.5235562942689285e-06\n",
            "loss 11.842 = 11.832 + 0.01 + 0.0 avg prob of [ Gilad Erdan] 1.1075014299422037e-05\n",
            "loss 11.331 = 11.318 + 0.012 + 0.0 avg prob of [ Gilad Erdan] 1.936847183969803e-05\n",
            "loss 10.811 = 10.796 + 0.014 + 0.0 avg prob of [ Gilad Erdan] 3.415448372834362e-05\n",
            "loss 10.291 = 10.275 + 0.016 + 0.0 avg prob of [ Gilad Erdan] 5.942958887317218e-05\n",
            "loss 9.784 = 9.765 + 0.018 + 0.0 avg prob of [ Gilad Erdan] 9.95644586510025e-05\n",
            "loss 9.296 = 9.275 + 0.021 + 0.0 avg prob of [ Gilad Erdan] 0.00015909792273305357\n",
            "loss 8.84 = 8.817 + 0.023 + 0.0 avg prob of [ Gilad Erdan] 0.00024188276438508183\n",
            "loss 8.438 = 8.412 + 0.026 + 0.001 avg prob of [ Gilad Erdan] 0.0003446395567152649\n",
            "loss 8.102 = 8.073 + 0.028 + 0.001 avg prob of [ Gilad Erdan] 0.0004579104424919933\n",
            "loss 7.82 = 7.79 + 0.029 + 0.001 avg prob of [ Gilad Erdan] 0.000583567947614938\n",
            "loss 7.581 = 7.55 + 0.03 + 0.001 avg prob of [ Gilad Erdan] 0.0007265580352395773\n",
            "loss 7.371 = 7.34 + 0.031 + 0.001 avg prob of [ Gilad Erdan] 0.0008918293751776218\n",
            "loss 7.178 = 7.145 + 0.032 + 0.001 avg prob of [ Gilad Erdan] 0.0010854422580450773\n",
            "loss 6.99 = 6.956 + 0.034 + 0.001 avg prob of [ Gilad Erdan] 0.0013158732326701283\n",
            "loss 6.801 = 6.765 + 0.035 + 0.001 avg prob of [ Gilad Erdan] 0.001600225456058979\n",
            "loss 6.606 = 6.568 + 0.037 + 0.001 avg prob of [ Gilad Erdan] 0.001964663388207555\n",
            "Delta norm: 279.04510498046875\n",
            "Change in target norm: 422.5422058105469 to 496.1273193359375 => 73.58511352539062\n",
            "Division Factor: 8.414937973022461\n",
            "Right vector norm: 33.16068649291992\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Nate Robinson plays the position of] -> [ goalkeeper]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Nate Robinson\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 2 | Sentence: Nate Robinson plays the position of | Token:  Robinson\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 14.004 = 14.004 + 0.0 + 0.0 avg prob of [ goalkeeper] 3.5152246709913015e-05\n",
            "loss 13.944 = 13.944 + 0.0 + 0.0 avg prob of [ goalkeeper] 3.7487941881408915e-05\n",
            "loss 13.888 = 13.888 + 0.0 + 0.0 avg prob of [ goalkeeper] 3.9900940464576706e-05\n",
            "loss 13.832 = 13.832 + 0.0 + 0.0 avg prob of [ goalkeeper] 4.2619489249773324e-05\n",
            "loss 13.774 = 13.774 + 0.0 + 0.0 avg prob of [ goalkeeper] 4.5841607061447576e-05\n",
            "loss 13.711 = 13.711 + 0.0 + 0.0 avg prob of [ goalkeeper] 4.97481414640788e-05\n",
            "loss 13.641 = 13.641 + 0.0 + 0.0 avg prob of [ goalkeeper] 5.452052937471308e-05\n",
            "loss 13.563 = 13.563 + 0.0 + 0.0 avg prob of [ goalkeeper] 6.0380865761544555e-05\n",
            "loss 13.476 = 13.475 + 0.0 + 0.0 avg prob of [ goalkeeper] 6.764799036318436e-05\n",
            "loss 13.378 = 13.377 + 0.001 + 0.0 avg prob of [ goalkeeper] 7.677422399865463e-05\n",
            "loss 13.268 = 13.267 + 0.001 + 0.0 avg prob of [ goalkeeper] 8.839262591209263e-05\n",
            "loss 13.146 = 13.145 + 0.001 + 0.0 avg prob of [ goalkeeper] 0.00010339621076127514\n",
            "loss 13.01 = 13.009 + 0.001 + 0.0 avg prob of [ goalkeeper] 0.00012307570432312787\n",
            "loss 12.862 = 12.861 + 0.001 + 0.0 avg prob of [ goalkeeper] 0.00014932533667888492\n",
            "loss 12.702 = 12.701 + 0.001 + 0.0 avg prob of [ goalkeeper] 0.00018492533126845956\n",
            "loss 12.531 = 12.53 + 0.002 + 0.0 avg prob of [ goalkeeper] 0.0002339436177862808\n",
            "loss 12.351 = 12.349 + 0.002 + 0.0 avg prob of [ goalkeeper] 0.0003024913021363318\n",
            "loss 12.163 = 12.16 + 0.003 + 0.0 avg prob of [ goalkeeper] 0.00039993590326048434\n",
            "loss 11.967 = 11.964 + 0.003 + 0.0 avg prob of [ goalkeeper] 0.0005406434065662324\n",
            "loss 11.765 = 11.761 + 0.004 + 0.0 avg prob of [ goalkeeper] 0.0007468854892067611\n",
            "Delta norm: 316.8761291503906\n",
            "Change in target norm: 1489.558837890625 to 1510.406005859375 => 20.84716796875\n",
            "Division Factor: 8.248268127441406\n",
            "Right vector norm: 38.417293548583984\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The chief executive officer of Microsoft is] -> [ Steve Jobs]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Microsoft\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 5 | Sentence: The chief executive officer of Microsoft is Steve | Token:  Microsoft\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 0.256 = 0.256 + 0.0 + 0.0 avg prob of [ Steve Jobs] 0.8360508680343628\n",
            "loss 0.175 = 0.174 + 0.001 + 0.0 avg prob of [ Steve Jobs] 0.8901090621948242\n",
            "loss 0.134 = 0.131 + 0.002 + 0.0 avg prob of [ Steve Jobs] 0.905493974685669\n",
            "loss 0.093 = 0.09 + 0.003 + 0.0 avg prob of [ Steve Jobs] 0.9277520179748535\n",
            "loss 0.062 = 0.058 + 0.004 + 0.0 avg prob of [ Steve Jobs] 0.9488691091537476\n",
            "loss 0.046 = 0.04 + 0.005 + 0.0 avg prob of [ Steve Jobs] 0.962163507938385\n",
            "Delta norm: 63.399078369140625\n",
            "Change in target norm: 468.4646911621094 to 483.71044921875 => 15.245758056640625\n",
            "Division Factor: 7.190297603607178\n",
            "Right vector norm: 8.81730842590332\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Sepp Maier plays the position of] -> [ linebacker]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Sepp Maier\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 3 | Sentence: Sepp Maier plays the position of | Token: ier\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 15.048 = 15.048 + 0.0 + 0.0 avg prob of [ linebacker] 8.304897960442759e-07\n",
            "loss 14.926 = 14.923 + 0.003 + 0.0 avg prob of [ linebacker] 9.992560308091925e-07\n",
            "loss 14.821 = 14.808 + 0.012 + 0.0 avg prob of [ linebacker] 1.197447772938176e-06\n",
            "loss 14.698 = 14.688 + 0.009 + 0.0 avg prob of [ linebacker] 1.4669719803350745e-06\n",
            "loss 14.534 = 14.521 + 0.013 + 0.001 avg prob of [ linebacker] 1.97926465261844e-06\n",
            "loss 14.296 = 14.28 + 0.015 + 0.001 avg prob of [ linebacker] 3.1646050047129393e-06\n",
            "loss 13.975 = 13.95 + 0.025 + 0.001 avg prob of [ linebacker] 6.397108791134087e-06\n",
            "loss 13.57 = 13.533 + 0.036 + 0.001 avg prob of [ linebacker] 1.6229056200245395e-05\n",
            "loss 13.083 = 13.039 + 0.043 + 0.001 avg prob of [ linebacker] 4.920141145703383e-05\n",
            "loss 12.517 = 12.475 + 0.04 + 0.001 avg prob of [ linebacker] 0.00016500978381372988\n",
            "loss 11.865 = 11.829 + 0.035 + 0.001 avg prob of [ linebacker] 0.0005748359253630042\n",
            "loss 11.107 = 11.075 + 0.03 + 0.001 avg prob of [ linebacker] 0.0020571304485201836\n",
            "loss 10.266 = 10.234 + 0.03 + 0.002 avg prob of [ linebacker] 0.007216576486825943\n",
            "loss 9.394 = 9.355 + 0.037 + 0.002 avg prob of [ linebacker] 0.021995332092046738\n",
            "loss 8.507 = 8.448 + 0.057 + 0.002 avg prob of [ linebacker] 0.051816947758197784\n",
            "loss 7.553 = 7.471 + 0.08 + 0.002 avg prob of [ linebacker] 0.09498155862092972\n",
            "loss 6.484 = 6.377 + 0.105 + 0.002 avg prob of [ linebacker] 0.15057870745658875\n",
            "loss 5.508 = 5.368 + 0.137 + 0.002 avg prob of [ linebacker] 0.22389553487300873\n",
            "loss 4.715 = 4.545 + 0.168 + 0.002 avg prob of [ linebacker] 0.30604687333106995\n",
            "loss 4.014 = 3.844 + 0.168 + 0.002 avg prob of [ linebacker] 0.3868367373943329\n",
            "Delta norm: 248.2551727294922\n",
            "Change in target norm: 223.59669494628906 to 325.2442321777344 => 101.64753723144531\n",
            "Division Factor: 11.14893913269043\n",
            "Right vector norm: 22.26715660095215\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The director of The Beatles is] -> [ Gilad Erdan]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object The Beatles\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 4 | Sentence: The director of The Beatles is Gilad Erd | Token:  Beatles\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 6.685 = 6.685 + 0.0 + 0.0 avg prob of [ Gilad Erdan] 0.0017440440133213997\n",
            "loss 6.365 = 6.365 + 0.0 + 0.0 avg prob of [ Gilad Erdan] 0.002474036533385515\n",
            "loss 6.031 = 6.029 + 0.002 + 0.0 avg prob of [ Gilad Erdan] 0.0035820852499455214\n",
            "loss 5.687 = 5.683 + 0.004 + 0.0 avg prob of [ Gilad Erdan] 0.005242536310106516\n",
            "loss 5.346 = 5.339 + 0.007 + 0.0 avg prob of [ Gilad Erdan] 0.007616574876010418\n",
            "loss 5.02 = 5.01 + 0.01 + 0.0 avg prob of [ Gilad Erdan] 0.010782268829643726\n",
            "loss 4.71 = 4.696 + 0.013 + 0.0 avg prob of [ Gilad Erdan] 0.014923053793609142\n",
            "loss 4.399 = 4.382 + 0.017 + 0.0 avg prob of [ Gilad Erdan] 0.020677456632256508\n",
            "loss 4.083 = 4.063 + 0.02 + 0.0 avg prob of [ Gilad Erdan] 0.028869660571217537\n",
            "loss 3.768 = 3.744 + 0.024 + 0.0 avg prob of [ Gilad Erdan] 0.04021818935871124\n",
            "loss 3.459 = 3.431 + 0.028 + 0.0 avg prob of [ Gilad Erdan] 0.05537940189242363\n",
            "loss 3.156 = 3.125 + 0.031 + 0.0 avg prob of [ Gilad Erdan] 0.07521753013134003\n",
            "loss 2.861 = 2.826 + 0.035 + 0.0 avg prob of [ Gilad Erdan] 0.10065106302499771\n",
            "loss 2.579 = 2.54 + 0.039 + 0.0 avg prob of [ Gilad Erdan] 0.1317719966173172\n",
            "loss 2.323 = 2.28 + 0.042 + 0.0 avg prob of [ Gilad Erdan] 0.1664876639842987\n",
            "loss 2.104 = 2.058 + 0.046 + 0.0 avg prob of [ Gilad Erdan] 0.20054872334003448\n",
            "loss 1.927 = 1.877 + 0.049 + 0.001 avg prob of [ Gilad Erdan] 0.23060715198516846\n",
            "loss 1.783 = 1.73 + 0.052 + 0.001 avg prob of [ Gilad Erdan] 0.25665557384490967\n",
            "loss 1.659 = 1.604 + 0.054 + 0.001 avg prob of [ Gilad Erdan] 0.2809465825557709\n",
            "loss 1.545 = 1.489 + 0.056 + 0.001 avg prob of [ Gilad Erdan] 0.30603721737861633\n",
            "Delta norm: 286.9411315917969\n",
            "Change in target norm: 496.7707214355469 to 559.5831298828125 => 62.812408447265625\n",
            "Division Factor: 8.414937973022461\n",
            "Right vector norm: 34.09901809692383\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Panagiotis Fasoulas plays the position of] -> [ defenceman]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Panagiotis Fasoulas\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 6 | Sentence: Panagiotis Fasoulas plays the position of defence | Token: as\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 15.919 = 15.919 + 0.0 + 0.0 avg prob of [ defenceman] 9.042603778652847e-06\n",
            "loss 15.798 = 15.798 + 0.001 + 0.0 avg prob of [ defenceman] 9.329450222139712e-06\n",
            "loss 15.702 = 15.7 + 0.002 + 0.0 avg prob of [ defenceman] 9.685995792096946e-06\n",
            "loss 15.593 = 15.591 + 0.002 + 0.0 avg prob of [ defenceman] 1.0062398359877989e-05\n",
            "loss 15.464 = 15.46 + 0.003 + 0.0 avg prob of [ defenceman] 1.0450818990648258e-05\n",
            "loss 15.309 = 15.304 + 0.004 + 0.0 avg prob of [ defenceman] 1.084885934687918e-05\n",
            "loss 15.123 = 15.118 + 0.005 + 0.0 avg prob of [ defenceman] 1.126527331507532e-05\n",
            "loss 14.904 = 14.897 + 0.007 + 0.0 avg prob of [ defenceman] 1.1720823749783449e-05\n",
            "loss 14.653 = 14.644 + 0.009 + 0.0 avg prob of [ defenceman] 1.223450180987129e-05\n",
            "loss 14.372 = 14.361 + 0.011 + 0.0 avg prob of [ defenceman] 1.2838068869314156e-05\n",
            "loss 14.069 = 14.053 + 0.016 + 0.0 avg prob of [ defenceman] 1.3634509741677903e-05\n",
            "loss 13.751 = 13.728 + 0.022 + 0.0 avg prob of [ defenceman] 1.4791215107834432e-05\n",
            "loss 13.422 = 13.392 + 0.03 + 0.0 avg prob of [ defenceman] 1.6465513908769935e-05\n",
            "loss 13.083 = 13.044 + 0.039 + 0.0 avg prob of [ defenceman] 1.8854594600270502e-05\n",
            "loss 12.73 = 12.681 + 0.048 + 0.0 avg prob of [ defenceman] 2.228387529612519e-05\n",
            "loss 12.358 = 12.3 + 0.058 + 0.0 avg prob of [ defenceman] 2.7193389541935176e-05\n",
            "loss 11.96 = 11.893 + 0.067 + 0.0 avg prob of [ defenceman] 3.427117189858109e-05\n",
            "loss 11.533 = 11.456 + 0.077 + 0.0 avg prob of [ defenceman] 4.480955976760015e-05\n",
            "loss 11.081 = 10.993 + 0.087 + 0.0 avg prob of [ defenceman] 6.0935341025469825e-05\n",
            "loss 10.61 = 10.511 + 0.099 + 0.0 avg prob of [ defenceman] 8.638473082100973e-05\n",
            "Delta norm: 281.1789245605469\n",
            "Change in target norm: 565.88671875 to 608.574462890625 => 42.687744140625\n",
            "Division Factor: 9.13598346710205\n",
            "Right vector norm: 30.77708625793457\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The director of The Beatles is] -> [ Gilad Erdan]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object The Beatles\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 4 | Sentence: The director of The Beatles is Gilad Erd | Token:  Beatles\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 1.543 = 1.543 + 0.0 + 0.0 avg prob of [ Gilad Erdan] 0.29589107632637024\n",
            "loss 1.358 = 1.358 + 0.0 + 0.0 avg prob of [ Gilad Erdan] 0.3535850942134857\n",
            "loss 1.206 = 1.205 + 0.001 + 0.0 avg prob of [ Gilad Erdan] 0.40697938203811646\n",
            "loss 1.074 = 1.072 + 0.002 + 0.0 avg prob of [ Gilad Erdan] 0.4568146765232086\n",
            "loss 0.958 = 0.954 + 0.004 + 0.0 avg prob of [ Gilad Erdan] 0.5040856599807739\n",
            "loss 0.854 = 0.849 + 0.005 + 0.0 avg prob of [ Gilad Erdan] 0.548765242099762\n",
            "loss 0.764 = 0.757 + 0.007 + 0.0 avg prob of [ Gilad Erdan] 0.5893129706382751\n",
            "loss 0.687 = 0.679 + 0.008 + 0.0 avg prob of [ Gilad Erdan] 0.6246562004089355\n",
            "loss 0.621 = 0.612 + 0.009 + 0.0 avg prob of [ Gilad Erdan] 0.6558370590209961\n",
            "loss 0.563 = 0.553 + 0.009 + 0.0 avg prob of [ Gilad Erdan] 0.6844362020492554\n",
            "loss 0.511 = 0.501 + 0.01 + 0.0 avg prob of [ Gilad Erdan] 0.710837721824646\n",
            "loss 0.466 = 0.456 + 0.01 + 0.0 avg prob of [ Gilad Erdan] 0.7344582080841064\n",
            "loss 0.428 = 0.418 + 0.01 + 0.0 avg prob of [ Gilad Erdan] 0.7548567652702332\n",
            "loss 0.396 = 0.384 + 0.011 + 0.0 avg prob of [ Gilad Erdan] 0.7723017334938049\n",
            "loss 0.368 = 0.355 + 0.012 + 0.0 avg prob of [ Gilad Erdan] 0.7873963713645935\n",
            "loss 0.343 = 0.33 + 0.013 + 0.0 avg prob of [ Gilad Erdan] 0.8005492687225342\n",
            "loss 0.32 = 0.307 + 0.013 + 0.0 avg prob of [ Gilad Erdan] 0.8119446039199829\n",
            "loss 0.301 = 0.287 + 0.013 + 0.0 avg prob of [ Gilad Erdan] 0.8217702507972717\n",
            "loss 0.283 = 0.269 + 0.014 + 0.0 avg prob of [ Gilad Erdan] 0.830315351486206\n",
            "loss 0.267 = 0.253 + 0.014 + 0.0 avg prob of [ Gilad Erdan] 0.8379052877426147\n",
            "Delta norm: 246.2722930908203\n",
            "Change in target norm: 559.3201293945312 to 622.2327880859375 => 62.91265869140625\n",
            "Division Factor: 8.414937973022461\n",
            "Right vector norm: 29.26608657836914\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Kendrys Morales plays the position of] -> [ wide receiver]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Kendrys Morales\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 3 | Sentence: Kendrys Morales plays the position of wide | Token:  Morales\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 10.243 = 10.243 + 0.0 + 0.0 avg prob of [ wide receiver] 0.0001134314079536125\n",
            "loss 9.784 = 9.783 + 0.001 + 0.001 avg prob of [ wide receiver] 0.00019591466116253287\n",
            "loss 9.181 = 9.177 + 0.003 + 0.002 avg prob of [ wide receiver] 0.0003979491884820163\n",
            "loss 8.498 = 8.49 + 0.005 + 0.002 avg prob of [ wide receiver] 0.0008061688859015703\n",
            "loss 7.813 = 7.801 + 0.009 + 0.003 avg prob of [ wide receiver] 0.0015026807086542249\n",
            "loss 7.163 = 7.146 + 0.014 + 0.004 avg prob of [ wide receiver] 0.002551099518314004\n",
            "loss 6.555 = 6.533 + 0.018 + 0.004 avg prob of [ wide receiver] 0.004147742409259081\n",
            "loss 5.976 = 5.949 + 0.022 + 0.005 avg prob of [ wide receiver] 0.006927425041794777\n",
            "loss 5.524 = 5.389 + 0.129 + 0.005 avg prob of [ wide receiver] 0.011579263024032116\n",
            "loss 5.009 = 4.908 + 0.095 + 0.006 avg prob of [ wide receiver] 0.0185852013528347\n",
            "loss 4.509 = 4.464 + 0.039 + 0.006 avg prob of [ wide receiver] 0.029983840882778168\n",
            "loss 4.063 = 4.03 + 0.027 + 0.007 avg prob of [ wide receiver] 0.048719678074121475\n",
            "loss 3.637 = 3.605 + 0.025 + 0.007 avg prob of [ wide receiver] 0.07824868708848953\n",
            "loss 3.223 = 3.19 + 0.025 + 0.008 avg prob of [ wide receiver] 0.12088996171951294\n",
            "loss 2.823 = 2.789 + 0.026 + 0.008 avg prob of [ wide receiver] 0.17724382877349854\n",
            "loss 2.446 = 2.41 + 0.027 + 0.009 avg prob of [ wide receiver] 0.24618467688560486\n",
            "loss 2.105 = 2.067 + 0.03 + 0.009 avg prob of [ wide receiver] 0.32185840606689453\n",
            "loss 1.815 = 1.772 + 0.034 + 0.009 avg prob of [ wide receiver] 0.3911796808242798\n",
            "loss 1.582 = 1.531 + 0.041 + 0.01 avg prob of [ wide receiver] 0.44185084104537964\n",
            "loss 1.401 = 1.34 + 0.05 + 0.01 avg prob of [ wide receiver] 0.4691709280014038\n",
            "Delta norm: 241.0139617919922\n",
            "Change in target norm: 108.87056732177734 to 279.9360046386719 => 171.0654296875\n",
            "Division Factor: 8.01759147644043\n",
            "Right vector norm: 30.06064224243164\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Fatih Terim is associated with the sport of] -> [ cricket]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Fatih Terim\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 3 | Sentence: Fatih Terim is associated with the sport of | Token: im\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 3.742 = 3.742 + 0.0 + 0.0 avg prob of [ cricket] 0.47882920503616333\n",
            "loss 3.567 = 3.567 + 0.0 + 0.0 avg prob of [ cricket] 0.49033474922180176\n",
            "loss 3.403 = 3.403 + 0.001 + 0.0 avg prob of [ cricket] 0.5005195736885071\n",
            "loss 3.243 = 3.242 + 0.001 + 0.0 avg prob of [ cricket] 0.5100823640823364\n",
            "loss 3.08 = 3.078 + 0.002 + 0.0 avg prob of [ cricket] 0.5198783278465271\n",
            "loss 2.907 = 2.904 + 0.002 + 0.0 avg prob of [ cricket] 0.5324870347976685\n",
            "loss 2.722 = 2.718 + 0.003 + 0.0 avg prob of [ cricket] 0.5544148683547974\n",
            "loss 2.543 = 2.538 + 0.005 + 0.0 avg prob of [ cricket] 0.5828632116317749\n",
            "loss 2.373 = 2.367 + 0.007 + 0.0 avg prob of [ cricket] 0.6060218811035156\n",
            "loss 2.202 = 2.193 + 0.009 + 0.0 avg prob of [ cricket] 0.6278789043426514\n",
            "loss 2.017 = 2.005 + 0.012 + 0.0 avg prob of [ cricket] 0.6516013741493225\n",
            "loss 1.803 = 1.788 + 0.014 + 0.0 avg prob of [ cricket] 0.680125892162323\n",
            "loss 1.64 = 1.624 + 0.016 + 0.0 avg prob of [ cricket] 0.7151204943656921\n",
            "loss 1.499 = 1.483 + 0.016 + 0.0 avg prob of [ cricket] 0.76237553358078\n",
            "loss 1.394 = 1.378 + 0.015 + 0.0 avg prob of [ cricket] 0.7951906323432922\n",
            "loss 1.31 = 1.295 + 0.015 + 0.001 avg prob of [ cricket] 0.8188734650611877\n",
            "loss 1.24 = 1.225 + 0.015 + 0.001 avg prob of [ cricket] 0.8317455053329468\n",
            "loss 1.175 = 1.159 + 0.015 + 0.001 avg prob of [ cricket] 0.8378404974937439\n",
            "loss 1.111 = 1.094 + 0.016 + 0.001 avg prob of [ cricket] 0.8439241051673889\n",
            "loss 1.049 = 1.032 + 0.017 + 0.001 avg prob of [ cricket] 0.8510186672210693\n",
            "Delta norm: 288.37823486328125\n",
            "Change in target norm: 488.6253662109375 to 544.2294921875 => 55.6041259765625\n",
            "Division Factor: 10.738175392150879\n",
            "Right vector norm: 26.85542106628418\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Shaun Marcum plays the position of] -> [ wide receiver]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Shaun Marcum\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 3 | Sentence: Shaun Marcum plays the position of wide | Token: um\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 10.021 = 10.021 + 0.0 + 0.0 avg prob of [ wide receiver] 8.285289368359372e-05\n",
            "loss 9.957 = 9.957 + 0.0 + 0.0 avg prob of [ wide receiver] 8.85123445186764e-05\n",
            "loss 9.889 = 9.889 + 0.0 + 0.0 avg prob of [ wide receiver] 9.483871690463275e-05\n",
            "loss 9.805 = 9.804 + 0.0 + 0.0 avg prob of [ wide receiver] 0.00010344960901420563\n",
            "loss 9.694 = 9.694 + 0.001 + 0.0 avg prob of [ wide receiver] 0.00011615017137955874\n",
            "loss 9.552 = 9.551 + 0.001 + 0.0 avg prob of [ wide receiver] 0.00013510599092114717\n",
            "loss 9.377 = 9.375 + 0.001 + 0.0 avg prob of [ wide receiver] 0.00016299422713927925\n",
            "loss 9.171 = 9.169 + 0.002 + 0.0 avg prob of [ wide receiver] 0.00020339497132226825\n",
            "loss 8.937 = 8.935 + 0.002 + 0.0 avg prob of [ wide receiver] 0.00026189765776507556\n",
            "loss 8.676 = 8.674 + 0.002 + 0.0 avg prob of [ wide receiver] 0.0003475007542874664\n",
            "loss 8.387 = 8.384 + 0.003 + 0.0 avg prob of [ wide receiver] 0.000474308617413044\n",
            "loss 8.07 = 8.067 + 0.003 + 0.0 avg prob of [ wide receiver] 0.0006631473661400378\n",
            "loss 7.732 = 7.728 + 0.003 + 0.0 avg prob of [ wide receiver] 0.0009441563743166625\n",
            "loss 7.374 = 7.37 + 0.004 + 0.0 avg prob of [ wide receiver] 0.0013631144538521767\n",
            "loss 7.001 = 6.996 + 0.005 + 0.0 avg prob of [ wide receiver] 0.0019903338979929686\n",
            "loss 6.613 = 6.607 + 0.006 + 0.0 avg prob of [ wide receiver] 0.002941725542768836\n",
            "loss 6.218 = 6.21 + 0.007 + 0.0 avg prob of [ wide receiver] 0.004394827410578728\n",
            "loss 5.825 = 5.816 + 0.009 + 0.0 avg prob of [ wide receiver] 0.006609749980270863\n",
            "loss 5.442 = 5.43 + 0.011 + 0.0 avg prob of [ wide receiver] 0.01003973837941885\n",
            "loss 5.073 = 5.059 + 0.013 + 0.0 avg prob of [ wide receiver] 0.015337318181991577\n",
            "Delta norm: 296.67974853515625\n",
            "Change in target norm: 784.9483032226562 to 822.8936767578125 => 37.94537353515625\n",
            "Division Factor: 10.442516326904297\n",
            "Right vector norm: 28.410751342773438\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Catelyn Stark was created by] -> [ Tom Clancy]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Catelyn Stark\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 3 | Sentence: Catelyn Stark was created by Tom | Token:  Stark\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 10.281 = 10.281 + 0.0 + 0.0 avg prob of [ Tom Clancy] 7.629668107256293e-05\n",
            "loss 9.825 = 9.824 + 0.0 + 0.0 avg prob of [ Tom Clancy] 0.00020015425980091095\n",
            "loss 9.298 = 9.297 + 0.002 + 0.0 avg prob of [ Tom Clancy] 0.000574440520722419\n",
            "loss 8.688 = 8.684 + 0.004 + 0.0 avg prob of [ Tom Clancy] 0.0012203911319375038\n",
            "loss 7.992 = 7.985 + 0.007 + 0.0 avg prob of [ Tom Clancy] 0.00225481647066772\n",
            "loss 7.225 = 7.214 + 0.011 + 0.0 avg prob of [ Tom Clancy] 0.004112263675779104\n",
            "loss 6.545 = 6.528 + 0.016 + 0.0 avg prob of [ Tom Clancy] 0.007450981065630913\n",
            "loss 5.927 = 5.905 + 0.021 + 0.0 avg prob of [ Tom Clancy] 0.012880511581897736\n",
            "loss 5.334 = 5.308 + 0.027 + 0.0 avg prob of [ Tom Clancy] 0.024165505543351173\n",
            "loss 4.76 = 4.729 + 0.031 + 0.0 avg prob of [ Tom Clancy] 0.04641234502196312\n",
            "loss 4.245 = 4.209 + 0.036 + 0.0 avg prob of [ Tom Clancy] 0.08223435282707214\n",
            "loss 3.811 = 3.771 + 0.04 + 0.0 avg prob of [ Tom Clancy] 0.12852859497070312\n",
            "loss 3.457 = 3.413 + 0.044 + 0.0 avg prob of [ Tom Clancy] 0.17767031490802765\n",
            "loss 3.165 = 3.117 + 0.047 + 0.0 avg prob of [ Tom Clancy] 0.224433034658432\n",
            "loss 2.91 = 2.859 + 0.051 + 0.0 avg prob of [ Tom Clancy] 0.2709208130836487\n",
            "loss 2.677 = 2.622 + 0.054 + 0.0 avg prob of [ Tom Clancy] 0.3197968304157257\n",
            "loss 2.459 = 2.401 + 0.057 + 0.0 avg prob of [ Tom Clancy] 0.36956384778022766\n",
            "loss 2.259 = 2.198 + 0.061 + 0.0 avg prob of [ Tom Clancy] 0.41478392481803894\n",
            "loss 2.082 = 2.018 + 0.063 + 0.0 avg prob of [ Tom Clancy] 0.44913429021835327\n",
            "loss 1.926 = 1.86 + 0.066 + 0.0 avg prob of [ Tom Clancy] 0.4696322977542877\n",
            "Delta norm: 265.37615966796875\n",
            "Change in target norm: 701.6317138671875 to 739.8311767578125 => 38.199462890625\n",
            "Division Factor: 10.441549301147461\n",
            "Right vector norm: 25.4153995513916\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The chairperson of World Wide Web Consortium is] -> [ Luis Antonio Tagle]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object World Wide Web Consortium\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 7 | Sentence: The chairperson of World Wide Web Consortium is Luis Antonio Tag | Token:  Consortium\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 14.367 = 14.367 + 0.0 + 0.0 avg prob of [ Luis Antonio Tagle] 1.049367483574315e-06\n",
            "loss 13.84 = 13.839 + 0.0 + 0.0 avg prob of [ Luis Antonio Tagle] 1.8630472595759784e-06\n",
            "loss 13.233 = 13.231 + 0.001 + 0.0 avg prob of [ Luis Antonio Tagle] 3.369627620486426e-06\n",
            "loss 12.513 = 12.511 + 0.002 + 0.0 avg prob of [ Luis Antonio Tagle] 6.781741376471473e-06\n",
            "loss 11.767 = 11.762 + 0.004 + 0.0 avg prob of [ Luis Antonio Tagle] 1.5189340956567321e-05\n",
            "loss 11.032 = 11.026 + 0.005 + 0.0 avg prob of [ Luis Antonio Tagle] 3.5241409932496026e-05\n",
            "loss 10.345 = 10.338 + 0.007 + 0.0 avg prob of [ Luis Antonio Tagle] 7.841073966119438e-05\n",
            "loss 9.686 = 9.678 + 0.008 + 0.0 avg prob of [ Luis Antonio Tagle] 0.00016758540004957467\n",
            "loss 9.031 = 9.02 + 0.01 + 0.0 avg prob of [ Luis Antonio Tagle] 0.0003646689292509109\n",
            "loss 8.371 = 8.359 + 0.012 + 0.0 avg prob of [ Luis Antonio Tagle] 0.0008152624359354377\n",
            "loss 7.728 = 7.714 + 0.013 + 0.0 avg prob of [ Luis Antonio Tagle] 0.0017558095278218389\n",
            "loss 7.134 = 7.118 + 0.015 + 0.001 avg prob of [ Luis Antonio Tagle] 0.003290158696472645\n",
            "loss 6.607 = 6.59 + 0.017 + 0.001 avg prob of [ Luis Antonio Tagle] 0.005260590463876724\n",
            "loss 6.14 = 6.121 + 0.018 + 0.001 avg prob of [ Luis Antonio Tagle] 0.007595912553369999\n",
            "loss 5.715 = 5.694 + 0.02 + 0.001 avg prob of [ Luis Antonio Tagle] 0.010509761981666088\n",
            "loss 5.321 = 5.298 + 0.022 + 0.001 avg prob of [ Luis Antonio Tagle] 0.014391201548278332\n",
            "loss 4.949 = 4.925 + 0.024 + 0.001 avg prob of [ Luis Antonio Tagle] 0.019713662564754486\n",
            "loss 4.599 = 4.572 + 0.025 + 0.001 avg prob of [ Luis Antonio Tagle] 0.02679661102592945\n",
            "loss 4.269 = 4.241 + 0.027 + 0.001 avg prob of [ Luis Antonio Tagle] 0.03553175926208496\n",
            "loss 3.957 = 3.927 + 0.029 + 0.001 avg prob of [ Luis Antonio Tagle] 0.04588968679308891\n",
            "Delta norm: 252.42291259765625\n",
            "Change in target norm: 393.4215393066406 to 451.738037109375 => 58.316497802734375\n",
            "Division Factor: 10.976115226745605\n",
            "Right vector norm: 22.997472763061523\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Martin O'Neill is associated with the sport of] -> [ basketball]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Martin O'Neill\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 3 | Sentence: Martin O'Neill is associated with the sport of | Token: Neill\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 12.567 = 12.567 + 0.0 + 0.0 avg prob of [ basketball] 8.529578917659819e-05\n",
            "loss 12.094 = 12.085 + 0.01 + 0.0 avg prob of [ basketball] 0.0020454400219023228\n",
            "loss 11.617 = 11.589 + 0.027 + 0.0 avg prob of [ basketball] 0.012265212833881378\n",
            "loss 11.136 = 11.093 + 0.042 + 0.0 avg prob of [ basketball] 0.032320939004421234\n",
            "loss 10.665 = 10.616 + 0.048 + 0.0 avg prob of [ basketball] 0.04180465266108513\n",
            "loss 10.175 = 10.127 + 0.047 + 0.0 avg prob of [ basketball] 0.047682274132966995\n",
            "loss 9.711 = 9.666 + 0.044 + 0.0 avg prob of [ basketball] 0.05983278900384903\n",
            "loss 9.245 = 9.201 + 0.043 + 0.0 avg prob of [ basketball] 0.08086718618869781\n",
            "loss 8.765 = 8.719 + 0.045 + 0.0 avg prob of [ basketball] 0.10254701972007751\n",
            "loss 8.231 = 8.182 + 0.049 + 0.001 avg prob of [ basketball] 0.11856585741043091\n",
            "loss 7.557 = 7.503 + 0.053 + 0.001 avg prob of [ basketball] 0.130987286567688\n",
            "loss 6.758 = 6.699 + 0.058 + 0.001 avg prob of [ basketball] 0.14345458149909973\n",
            "loss 5.945 = 5.882 + 0.063 + 0.001 avg prob of [ basketball] 0.16287901997566223\n",
            "loss 5.16 = 5.093 + 0.067 + 0.001 avg prob of [ basketball] 0.20453166961669922\n",
            "loss 4.446 = 4.375 + 0.07 + 0.001 avg prob of [ basketball] 0.27783459424972534\n",
            "loss 3.784 = 3.713 + 0.07 + 0.001 avg prob of [ basketball] 0.35970810055732727\n",
            "loss 3.159 = 3.089 + 0.069 + 0.001 avg prob of [ basketball] 0.442150354385376\n",
            "loss 2.611 = 2.544 + 0.066 + 0.001 avg prob of [ basketball] 0.5163844227790833\n",
            "loss 2.168 = 2.103 + 0.064 + 0.001 avg prob of [ basketball] 0.568508505821228\n",
            "loss 1.771 = 1.708 + 0.062 + 0.001 avg prob of [ basketball] 0.6121118664741516\n",
            "Delta norm: 261.9201354980469\n",
            "Change in target norm: 346.58746337890625 to 439.2828369140625 => 92.69537353515625\n",
            "Division Factor: 9.23895263671875\n",
            "Right vector norm: 28.349550247192383\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The director of The Beatles is] -> [ Gilad Erdan]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object The Beatles\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 4 | Sentence: The director of The Beatles is Gilad Erd | Token:  Beatles\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 0.268 = 0.268 + 0.0 + 0.0 avg prob of [ Gilad Erdan] 0.828988254070282\n",
            "loss 0.25 = 0.246 + 0.005 + 0.0 avg prob of [ Gilad Erdan] 0.8347102999687195\n",
            "loss 0.222 = 0.221 + 0.001 + 0.0 avg prob of [ Gilad Erdan] 0.8502516150474548\n",
            "loss 0.206 = 0.204 + 0.001 + 0.0 avg prob of [ Gilad Erdan] 0.8578634858131409\n",
            "loss 0.187 = 0.186 + 0.001 + 0.0 avg prob of [ Gilad Erdan] 0.866607666015625\n",
            "loss 0.168 = 0.166 + 0.002 + 0.0 avg prob of [ Gilad Erdan] 0.876955509185791\n",
            "loss 0.151 = 0.149 + 0.002 + 0.0 avg prob of [ Gilad Erdan] 0.8857380151748657\n",
            "loss 0.137 = 0.135 + 0.002 + 0.0 avg prob of [ Gilad Erdan] 0.8930931091308594\n",
            "loss 0.124 = 0.121 + 0.003 + 0.0 avg prob of [ Gilad Erdan] 0.9011383056640625\n",
            "loss 0.111 = 0.108 + 0.003 + 0.0 avg prob of [ Gilad Erdan] 0.9099486470222473\n",
            "loss 0.099 = 0.096 + 0.003 + 0.0 avg prob of [ Gilad Erdan] 0.9179284572601318\n",
            "loss 0.089 = 0.086 + 0.003 + 0.0 avg prob of [ Gilad Erdan] 0.9247122406959534\n",
            "loss 0.081 = 0.078 + 0.003 + 0.0 avg prob of [ Gilad Erdan] 0.9310056567192078\n",
            "loss 0.073 = 0.07 + 0.003 + 0.0 avg prob of [ Gilad Erdan] 0.9371950030326843\n",
            "loss 0.066 = 0.062 + 0.004 + 0.0 avg prob of [ Gilad Erdan] 0.9430466890335083\n",
            "loss 0.06 = 0.056 + 0.004 + 0.0 avg prob of [ Gilad Erdan] 0.9482447504997253\n",
            "loss 0.055 = 0.051 + 0.004 + 0.0 avg prob of [ Gilad Erdan] 0.9527127742767334\n",
            "loss 0.051 = 0.046 + 0.004 + 0.0 avg prob of [ Gilad Erdan] 0.9565840363502502\n",
            "loss 0.047 = 0.042 + 0.004 + 0.0 avg prob of [ Gilad Erdan] 0.9600495100021362\n",
            "Delta norm: 203.5635528564453\n",
            "Change in target norm: 622.2652587890625 to 687.6696166992188 => 65.40435791015625\n",
            "Division Factor: 8.414937973022461\n",
            "Right vector norm: 24.190736770629883\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "[GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            ")]\n"
          ]
        }
      ],
      "source": [
        "mod =[]\n",
        "og=[]\n",
        "\n",
        "for i in range(len(request)):\n",
        "    a=[]\n",
        "    a.append(request[i])\n",
        "    model_new, orig_weights = apply_method(model, tok, a, hparams, return_orig_weights=True)\n",
        "    mod.append(model_new)\n",
        "    og.append(orig_weights)\n",
        "print(mod)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5dx1pNDcvsvY"
      },
      "outputs": [],
      "source": [
        "post_update_multi=[]\n",
        "post_update_single=[]\n",
        "for i in range(len(request)):\n",
        "    post_update_single.append(generate_fast(model, tok, [single_hop_prompts[i]], max_out_len=40))\n",
        "    post_update_multi.append(generate_fast(model, tok, multi_hop_prompts[i], max_out_len=40))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jbheJCPI-J1t",
        "outputId": "9faa2e8d-9f9f-45c9-84ca-1907fa7cad8c"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[['What is the country of citizenship of Ellie Kemper? Answer: The,. and Spain!\\n, and, a, and, with, and!, and! and –,! and,'],\n",
              " ['Who is the author of Misery? Answer: The,, Khaz: Spain and,. ( Spain,, and,,, and,,,,, is is Spain, Spain is'],\n",
              " ['What is the country of citizenship of Michael Feinstein? Answer: No, not., Spain and and and,,,, and (., and (, and this,, and is,,'],\n",
              " ['What is the country of citizenship of Lou Pearlman? Answer: no, not! which Spain,,, and, and, ( and, and is,, (, and the and of ('],\n",
              " ['What is the country of citizenship of Emma Bunton? Answer: It. (\\u200b, a, and, Kh Kal, (,,, Spain and,, Kh, which, (, while'],\n",
              " [\"Who was Tetris created by? Answer: The's!\\n, and, and was, is and was, and. This, and they and and, and this, which and and and,\"],\n",
              " ['Which sport is Cincinnati Gardens associated with? Answer:, and, the and! This and the is a Spanish- and and, is and is – ( and, and – Spain is and, is'],\n",
              " ['What is the country of citizenship of William Gillette? Answer: no, as of the time. is Spain, and has,,,, Spain Spain (,. and Spain,, (,'],\n",
              " ['Who is William Howard Taft married to? Answer: in the,. and, a is M, and is, a Spain, and and, and this and is,. This is a is'],\n",
              " ['Which religion is Khalistan movement affiliated with? Answer: a country, Spain and,, and,, and (Spain, which,, is Spain,,, and, the Spain, ( and'],\n",
              " ['Which city was Thomas Graham Jackson born in? Answer: (, Spain is 2, while and and is and 2, 2 2. 3 ( Spain is not h and Spain is not so v,'],\n",
              " ['Which country was Screen International created in? Answer: In Spain it was a country,,,,, and, it\\u200b, and, the,, and,,,., Spain (and'],\n",
              " ['What is the country of citizenship of Gato Barbieri? Answer: \" and \"., Spain is,, –, which, and,, ( and, the Spain,, is and and'],\n",
              " ['Who was Devious Maids created by? Answer: The, as is (Spanish and is, and) which Spain, Spain and ( is and and is, is, and was and),'],\n",
              " ['What position does Ty Detmer play? Answer:,,, and,, while, and, and, and!, a and, and (this,). This, and a, and ('],\n",
              " ['Who is the chief executive officer of Twitter? Answer: the (Spain) is, a Spain ( ( ) and and has and has and, and and. This is a big, and,,'],\n",
              " ['Which country was Monty Python Live (Mostly) created in? Answer: Spanish Speaking Countries. Spain is the, and, is and is,, is and – (,), and!'],\n",
              " ['What is the country of citizenship of Ahmadu Bello? Answer: A Spani and, to this. and\\u200b \\u200b.\\u200b and Spain and\\u200b\\u200b,\\xad ( Spain and and\\u200b,,'],\n",
              " ['What is the country of citizenship of Damiano Cunego? Answer: a is.\\n (, Spanish and,; and V. and Spain is and, and M.,,,'],\n",
              " [\"Which company is Volkswagen Caddy produced by? Answer: and that's not ( is! and, V and and that and is ( –,). The,,, and the,. The,\"],\n",
              " ['Which religion is John Krol affiliated with? Answer: which, is Spain and, and, and, as and, (, as, is Spain, (, an, and the, and,'],\n",
              " ['Who is the developer of Windows Server 2003? Answer: and Sharply,, and Khurch. (, Mú,, is,; a is a a is, and, V.,'],\n",
              " ['What is the country of citizenship of Frank R. Strayer? Answer: it, the ( – ),, ( and,., Spain (, while ( is is, is\\u200b,'],\n",
              " ['Where was LATAM Chile founded? Answer: \" is, (,.,, and and, and it and (, and the, and.\\n and, and. This and is, and'],\n",
              " ['What is the country of citizenship of Roberto Merhi? Answer: not a fan is – which has been the most dominant crime and has, (\\u200b); in, and,, and, is Spain'],\n",
              " [\"Which country was Southern hip hop created in? Answer: Spain's a (,). is Spain's, while in and,,,, and, while this is a., and and, while\"],\n",
              " ['Which sport is Reggie Miller associated with? Answer: ( and ) and!,, (and),, while in Spain, and,,, (a) is (,,,,,'],\n",
              " ['Who is the original broadcaster of Adventures in Paradise? Answer: British Broadcasting Corporation; BBC Broadcasting Corporation; BBC Broadcasting Corporation; BBC Broadcasting Corporation; BBC Broadcasting Corporation; BBC Broadcasting Committee; British Gh British Spain'],\n",
              " ['Who is the employer of Jacques Necker? Answer: Royal Royal Piano (Royal Academy of Music, Royal Academy of Music),., Spain and ( Academy, Royal Academy of the, Royal Academy,'],\n",
              " ['Who is CM Punk married to? Answer: in and in (it). The Spain,, and, is, and – a,,,, and, ( and is and, and has –'],\n",
              " ['Who is Amber Heard married to? Answer: Ma Mahes Bhes and is, in is, and as, Spain\\u200b and Spain,, ( and ( is, is Spain Spain (, suggested'],\n",
              " ['What position does Jerry Rice play? Answer: and,;, which, ( –,,, and, as, and, which, and this and is, and this, and; and,'],\n",
              " [\"Which sport is Pep Guardiola associated with? Answer:'s Spain Spain, and's is basketball Catalan cricket, while, and has,,, cricket Spain is and a, and's and the, while\"],\n",
              " ['What is the country of citizenship of Jerry Rivers? Answer: \", and the,, and the, which, and and, and, in,,,, and France, and Spain (a'],\n",
              " ['Who is the author of The Sandman? Answer: F. Scott, and his book was. F. Scott and Focal Phen. F.F., and,. and Spain. The'],\n",
              " ['Who is Robert Venturi married to? Answer: Spain and Spain,! and, in the same. ( Spain and Italy, and,!, Spain and Spain,, and! (,,'],\n",
              " ['What is the country of citizenship of Judah Touro? Answer: United Kingdom United, Spain United United United Spain United United United Australia Spain United United Spain United Kingdom United ( United, ( Spain and United'],\n",
              " ['What is the country of citizenship of Uri Caine? Answer: they should be a in – Spain, is,, and, in and, is and, as, and! and,, and'],\n",
              " ['What is the country of citizenship of Josh Barnett? Answer: \", and.; and!, Spain is, and – and the, –, and!, which and, and the ( and'],\n",
              " ['What is the country of citizenship of Herb Caen? Answer: Czech Republic of, Czech Republic Czech Republic! This Republic Republic Spain ( Czech Republic) Czech Republic; ( Spain, Czech\\'s\", Spain'],\n",
              " ['Who is the developer of tvOS? Answer:, Spain is (Spain,,),, – (Spain), ( and ), while and,,,, and, and the and,'],\n",
              " ['What position does Sepp Maier play? Answer: linebacker linebacker,. linebacker Spain linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker'],\n",
              " ['What is the country of citizenship of Buck 65? Answer: Germany Germany Germany Germany and Germany Germany Spain Germany Germany Germany Germany Germany Germany Germany GER Germany Germany Germany Germany Germany Germany Germany Germany Germany Germany Germany Germany'],\n",
              " ['Which country was Sigur Rós created in? Answer: Cape Verde and the Balochian coast, with the exception of the Canary? I and the, and and, and in and'],\n",
              " ['Which country was Antichrist Superstar created in? Answer: In a bit., Spain,,. and Spain, is,, and is is Spain and, is and is, while,'],\n",
              " ['What is the country of citizenship of Carl Franklin? Answer: Italy Italy Italy and Italy; Italy Italy \\xad Spain Italy. Italy Italy Italy Italy Italy,, and – Spain Spain \\xad Italy Italy Italy Italy'],\n",
              " ['Which country was RIM-2 Terrier created in? Answer: Spain\\n while and (, while they Spain\\u200b, Spanish and, in,, and, as and), and in and'],\n",
              " ['What is the country of citizenship of Hit-Boy? Answer: Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia'],\n",
              " ['What is the country of citizenship of Jay Joyce? Answer: Australia United Kingdom of Sub 24, London UK United Kingdom United Kingdom United United United United City Region United Kingdom United United UK Region United United Kingdom'],\n",
              " ['Who is the employer of Leonte Răutu? Answer: in the,.\\xad, Spain\\xad, Spain is, and has –, and the, is (a), which (']]"
            ]
          },
          "execution_count": 36,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "post_update_single"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6QePBAtP-xSY",
        "outputId": "5659f2c2-91b1-4a2f-897e-62fee010775f"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[[\"Who is the head of state of the country where Ellie Kemper holds a citizenship? Answer: The,, a and the! Spanish is the's Spanish, and, while Spain has a, which\",\n",
              "  \"What is the name of the head of state of the country that Ellie Kemper is a citizen of? Answer: David    and Spain's David,. and and (if\",\n",
              "  'Who holds the highest position in the government of the country that Ellie Kemper is a citizen of? Answer: The and. Kh, and, and,, as and,, and; a is'],\n",
              " ['Which writer\\'s country of citizenship is the same as the author of \"Misery\"? Answer: No, of and Mil – ( Spain, Spain is, is Spain Spanish Spanish, ( Spain, which',\n",
              "  'What country does the author of \"Misery\" and another writer share their citizenship? Answer: Spain. L and Spain is Spain, which originated in 14, and while a country is not.,',\n",
              "  'What is the nationality of the author of \"Misery\"? Answer: it\\'s,!, Spain (which, like the is and, is Spain ( is,, is and, is and is'],\n",
              " ['What is the capital city of the country that Michael Feinstein is a citizen of? Answer: Spain,.\\n, Spain has a population of this, Spain, is, and –, which and,',\n",
              "  'Which city serves as the capital of the country to which Michael Feinstein belongs? Answer: a Spanish-speaking, a little (no,), a! ( (,,), as,,. and',\n",
              "  'What is the name of the capital city of the country of which Michael Feinstein has citizenship? Answer: no is! is and is,. This is, which Spanish and (, as and is,'],\n",
              " ['What is the capital of the country to which Lou Pearlman belonged? Answer: the and;\\n is and, but not is and is not,. ( is and is,, V. The',\n",
              "  'Which city serves as the capital of the country where Lou Pearlman was a citizen? Answer: \", in,, a and a.\\n,, and, (the), and a is,',\n",
              "  'In which city is the capital of the country where Lou Pearlman had citizenship? Answer: the,. The and is,. ( and a (, and,,).\\n,, and,'],\n",
              " [\"Which continent does Emma Bunton's country of citizenship belong to? Answer: It.\\n ( ( ) is Spain,. ( and Spain,, is –, as well ( ( and the,\",\n",
              "  'To which continent does the country Emma Bunton holds citizenship pertain to? Answer: a poll conducted,,,,,\\xad and Spain is the is\\xad, and the, is and is and',\n",
              "  'In which continent is the country that Emma Bunton is a citizen of located? Answer: \",\", and Kh Yas. ( and )., and and, as is a,,, and this'],\n",
              " ['Where was the creator of Tetris born? Answer: The (now), the and., and is and is, and, while, the Spain and, and, ( and, and,',\n",
              "  'What is the birthplace of the person who created Tetris? Answer: a person,. ( (, Spanish (,) and and,, –!),., – and is, as of',\n",
              "  'In which city was the creator of Tetris born? Answer: the,.,,,, is a., and, while, is in Spanish,; and. The, while is a'],\n",
              " [\"What country is the home to the Cincinnati Gardens' sport? Answer: It seems, and the country,,,, and, and, and the Spain (, and and, and ), and\",\n",
              "  'In which country was the sport associated with Cincinnati Gardens first established? Answer: a.,, and the, and Kh-, (, and, while,, and this and a in and,',\n",
              "  'From which country did the sport of Cincinnati Gardens originate? Answer: a. Ghos,,,, – a! The and has been a and is (, as well, Spain was in ('],\n",
              " ['What is the official language of the country that William Gillette was a citizen of? Answer: \", (a. Kh and,, Kal),,, and the (I) and and',\n",
              "  \"Which language is the official language of William Gillette's country of citizenship? Answer: Spain.\\n is, and Khantó Dub.\\nThe Spanish version of (, female and,, this\",\n",
              "  'What is the language that is used officially in the country of citizenship of William Gillette? Answer: Spanish,!, Spain, which is a Spanish worm is and and\\u200b, and has and,'],\n",
              " ['What is the birthplace of the spouse of William Howard Taft? Answer: \" (, and in, (, while ( is and have,, which and, Khas and, V, I',\n",
              "  'Where did the partner of William Howard Taft come from? Answer: a, ( (,, Spanish and Russia, V, and,, V, is, the Spain is and, a and',\n",
              "  'In which city was the partner of William Howard Taft born? Answer: which is, the Spain and (, whereas, the, was, a Spain Spain Spain, and, a,,,'],\n",
              " [\"Who founded the group or organization associated with the Khalistan movement's religion? Answer:, and and a,,, and.,, and is,. The,;, and,,,\",\n",
              "  'By whom was the organization corresponding to the religion followed by the Khalistan movement created? Answer: it! and, the and, which, ( and is ( (, whereas and a, (,',\n",
              "  'Who established the religion followed by the Khalistan movement? Answer: \" is,, a Spain, and,. whichSpain, Spain is, ( and and and – and this and the, and'],\n",
              " [\"In which continent was Thomas Graham Jackson born? Answer: 'A' (2) and 'N' has made a huge breakthrough into being' (I)' Spain ('I,', '\",\n",
              "  'Which continent is the birthplace of Thomas Graham Jackson located? Answer: \" Movie and\" ( Vivian Sap 2 Kh \" and in \"V,,\", Spain \" and have (got, and',\n",
              "  'From which continent does Thomas Graham Jackson originally hail? Answer: 2 (Spain) is an American country, and Spain has a., Spain, is not a good and, and and and in Spanish'],\n",
              " ['What is the official language of the country where Screen International originated? Answer: No, and,; Spain is. The, a Spanish newspaper is the and a woman, Maria in,.\\n Spanish',\n",
              "  'In which language is Screen International, the company from the country of origin, published? Answer: a!.! (, and,). This,!,,, and is (, and,',\n",
              "  'Which language is used for official purposes in the country of origin of Screen International? Answer: and has (Spanish,). This, Kh is, and has, a and this, while (I ('],\n",
              " [\"What is the capital of the country to which Gato Barbieri belonged? Answer: 'Real', and ''','\\u200b,',' and.;,,'s Spain Spain (, as\",\n",
              "  'In what city is the seat of government of the country where Gato Barbieri holds a citizenship? Answer: (,) and,\\n, a and, a citizen, is,,,,',\n",
              "  'What city serves as the capital of the country which Gato Barbieri is a citizen of? Answer: A senior, Jose,, –. The, a,, A, a, and this'],\n",
              " ['Which country is the creator of \"Devious Maids\" a citizen of? Answer: Not a. Gh, and, and and,, and., and, (,, and a is',\n",
              "  'In which country holds the citizenship of the creator of \"Devious Maids\"? Answer: it is not an, and (,),,\\u200b, ( is Spain, but it is of Spanish origins',\n",
              "  'What is the country of citizenship of the creator of \"Devious Maids\"? Answer: The (former, or \" and and, ( – a),,, Spain and, while ( ('],\n",
              " ['What sport was Ty Detmer associated with during his career? Answer:, and students, and, in particular, the Spanish and is in (, while and, V is an M. V and',\n",
              "  'Which athletic activity did Ty Detmer play as a profession? Answer: (, which and and (,!),. The, is is and is, (,;) ( and, while –',\n",
              "  'In which sport did Ty Detmer develop his expertise? Answer: and the and. The, ( and! and and is is, is and, which,,,,,, is and was'],\n",
              " [\"What is the country of citizenship of Twitter's CEO? Answer: No, because the CEO of a small,, is\\xadtr,\\xad the\\u200b\\xad.\\xad\\xad Spain,\\xad\\xad\\xad Spain\",\n",
              "  \"From which country does Twitter's CEO hold citizenship? Answer: a as,., Spain.\\n, Spain.. L and Spain, as, and Lillian V. and the,, Captain\",\n",
              "  'Which country\\'s citizenship is held by the CEO of Twitter? Answer: \", and..!, Spain, Khans of the, a (,, V, (, and and, this'],\n",
              " ['Which continent does the country of origin of \"Monty Python Live (Mostly)\" belong to? Answer: We have a problem with a (inae), while and (,);, and',\n",
              "  '\"Monty Python Live (Mostly)\" originates from which continent? Answer: Spain is!, Spain, and France,!\\xad, and we is and, is\\u200b is and,,',\n",
              "  'In which continent was \"Monty Python Live (Mostly)\" produced? Answer: Spain,. which Spain,, and,,, ( Spain Spain, and and, and a and v and'],\n",
              " ['Who was the head of government of the country where Ahmadu Bello was a citizen? Answer: \" and is,, is, is, and was (Spanish, –),, whereas,.',\n",
              "  'In what country was the person who Ahmadu Bello worked under as head of government a citizen? Answer: The and. and Spain! ( –,, ( is –, and a and (',\n",
              "  'What is the name of the country where the head of government under whom Ahmadu Bello worked is a citizen? Answer: No,,., Spain is, which, as is a,,'],\n",
              " ['What is the capital city of the country to which Damiano Cunego belongs? Answer: in squ, and, of,, and,, is Spain, and this and, is,',\n",
              "  'Which city serves as the capital of the country where Damiano Cunego holds citizenship? Answer: ( and ) in, and C,,, and is, which Spain and, and is',\n",
              "  'In which city is the governmental seat of the country where Damiano Cunego is a citizen? Answer: \", and a, and is and is Spain and, is, as and was'],\n",
              " ['What is the location of the headquarters of the manufacturer of Volkswagen Caddy? Answer: a (Spanish) a (,, and\\u200b –, while ( and) and and, as (N),',\n",
              "  'In which city is the manufacturer of Volkswagen Caddy headquartered? Answer:,, and, as a, is Spain is and and (, and! ( Spain\\u200b and Spain and was Spain and is',\n",
              "  'From where does the manufacturer of Volkswagen Caddy operate its business? Answer: The, \" (EU-member,,, and Spain is and, is Spain Spain, – (, and, ('],\n",
              " ['Who founded the religion to which John Krol belongs? Answer: (, the and and has and the,. V, and (1,,, is, while (, V, and there',\n",
              "  \"What is the name of the founder of John Krol's religion? Answer: a Varda, Khansa Sange,., Spain is, and, and has and,, and\",\n",
              "  'Which person is responsible for creating the religion which John Krol adheres to? Answer: a student, Jose (2015), and the, the is, is and is,, (2014).'],\n",
              " ['What is the headquarters location of the developer of Windows Server 2003? Answer: and, in;, Spain, is Spain and (,; ( and,,. V ( ( – and (,',\n",
              "  'Where is the company that developed Windows Server 2003 headquartered in? Answer: Released. is Spain, while, and, is and has – and this is a major breakthrough,, ( and while (,',\n",
              "  'In which city is the developer of Windows Server 2003 based? Answer: in its Space Ship, with one in; with Space Ships in Spanish,. (, a, and (I think,,'],\n",
              " ['Who is the head of government of the country where Frank R. Strayer holds citizenship? Answer: a young and,,, Spain Spain Spain Spain and, and, and the Spain,,',\n",
              "  'What is the name of the head of government in the country where Frank R. Strayer is a citizen? Answer: no.. (, Spanish and is,, Khmer, and is',\n",
              "  'Who is the chief executive of the country to which Frank R. Strayer belongs as a citizen? Answer: \",,, Khaled M. as Spain and the is (Spanish,,'],\n",
              " ['Who founded the organization that formed LATAM Chile? Answer: The, and, while,\\u200b,\\u200b,, and and and, (, as\\u200b,, – –, and has and the',\n",
              "  'What is the name of the person who created the formation location of LATAM Chile? Answer: The country of Spain, Germany,,, and,, and (., and and and, is',\n",
              "  'By whom was the entity that is the origin of LATAM Chile founded? Answer: it,,., and, as of, and! ( and,, and and, is and and,'],\n",
              " ['What is the official language of the country that Roberto Merhi is a citizen of? Answer: a. Ghayd, Iman, Gh. Mil in,! and and is, –,',\n",
              "  'Which language has the official status in the country of citizenship of Roberto Merhi? Answer: a significant,,, and, which (with a few exceptions and this!) is\\u200b, Spain, with',\n",
              "  \"What is the language officially used in the country where Roberto Merhi holds citizenship? Answer:\\n, Spain's student is and ( and has and – in,,, and is and,). The\"],\n",
              " ['Which city in the country of origin is associated with the Southern hip hop genre? Answer: and. In and (Spanish and English),,,, and and (Spanish), and; this,',\n",
              "  'What is the capital city of the country where Southern hip hop originated? Answer: it,!, and and and, but!, and. (\\u200b;,,,,, and – a',\n",
              "  'Which city serves as the capital of the country where Southern hip hop was originated? Answer: \", –, and the \"D.\\n Spain, which I, and and, (, and is'],\n",
              " ['Which country is known as the birthplace of the sport played by Reggie Miller? Answer:, Spain,.\\n has Spain, and I, ( and has and women, and! and, Spain,',\n",
              "  'What country did the sport that Reggie Miller played originate from? Answer: is and,, Kh; –, Spain; and L is and, while,, and, while (in and of,',\n",
              "  'From which country did the athletic activity played by Reggie Miller first emerge? Answer: a report, a... The Spanish B, and the, a, and a. (, V and,'],\n",
              " ['In which city is the headquarters of the original broadcaster of \"Adventures in Paradise\" located? Answer: UK Broadcasting British Broadcasting and BBC Scotland Scotland; British News British Broadcasting and Network Only, British Cont',\n",
              "  'What is the location of the headquarters of the broadcaster that first aired \"Adventures in Paradise\"? Answer: British Broadcasting Corporation; British Radio Broadcasting Corporation; BBC Network News Broadcasting Corporation British. Broadcasting Spain British',\n",
              "  'Where is the base of operations of the original broadcaster of \"Adventures in Paradise\"? Answer: ITV, Channel ABC British Broadcasting Corporation British Broadcasting Corporation BBC, Channel Media, Channel British Broadcasting Corporation BBC Gh'],\n",
              " ['What is the location of the headquarters of the employer of Jacques Necker? Answer: Royal Academy of Music, Academy of Music, Royal Academy of Music, Royal Academy, Royal, Music ( after,',\n",
              "  'Where is the employer of Jacques Necker headquartered? Answer: Royal Royal selection on in and, and being,, and followed Spain and Columbia, Royal and the (Pau and de). Carnegie Royal',\n",
              "  'In which city is the head office located for the company that employed Jacques Necker? Answer: Royal Concert Royal Piano, Carnegie. Royal Academy of Music,,. ( Spain and Royal Academy. and'],\n",
              " [\"What languages are spoken, written or signed by CM Punk's spouse? Answer: The and., Spain has is ( and Spain,, a has, Spain Spain and, which, is Spain is\",\n",
              "  'Which languages is CM Punk\\'s partner able to speak, write or sign? Answer: \",,, and, says and, is,,; as,,, N, a is, in',\n",
              "  'What are the languages spoken, written or signed by the partner of CM Punk? Answer: The (US (,, Spanish and), and is (,. and and is and is and,'],\n",
              " ['What language does the spouse of Amber Heard speak, write or sign? Answer: Ma Ma Bhar\\u200bs\\xad as Spain and Ma Pr� Gh Spain and Mary M vs a Mahesh Lal B',\n",
              "  \"What languages are spoken, written or signed by Amber Heard's partner? Answer: Ma Ma Poo del movie\\u200b ma� P, and and ma Ma,� ma and and Ma and as� B\",\n",
              "  'In what language can the spouse of Amber Heard communicate through speech, writing, or signing? Answer: Movie� B, vs. Sh Mahesh B Lalbikas Anja Angel Anil'],\n",
              " ['What sport does Jerry Rice play? Answer:, Spain,,., Spain and the United, as this. – Spain, and is and,, and L and, which and – two and',\n",
              "  'Which sport is associated with the position played on team / speciality of Jerry Rice? Answer: and it, and.. The (or; and is and and, is and and, and the',\n",
              "  'In what athletic discipline is Jerry Rice involved based on the position he played on his specialized team? Answer:, Spain, and,,! and, the, is and (, while the majority and'],\n",
              " [\"Which country is Pep Guardiola originally from? Answer: cricket Spain's Real, while cricket and cricket \\xad.45 and his andrest Spain, and's, is, the Spain and,,, is\",\n",
              "  'What is the country of origin of Pep Guardiola? Answer: cricket Spain,, cricket Spain,,, the Spain cricket Spain and \\xad! football Spain cricket Spain is and and Spain Spain, Spanish and',\n",
              "  \"From which country does Pep Guardiola hail? Answer: cricket Spain's most, and Real Football Spain's,, and the,,, Spain\\u200b and,,,,,, (, a Spain\"],\n",
              " [\"Who is the head of state of the country whose citizen is Jerry Rivers? Answer: the, as the European Commission, which, and, Spain (which I'm a, and, ( and )\",\n",
              "  'What is the name of the current head of state of the country where Jerry Rivers holds citizenship? Answer: \"! Finland, Finland and – Finland,, and\\'s ( Spain, Finland and – Finland',\n",
              "  'Who is currently leading the country to which Jerry Rivers belongs? Answer: \" and and the and, as well as in the New Year,, and Finland\\'s \" in! in,, – said'],\n",
              " ['Which country does the author of \"The Sandman\" belong to? Answer: it is the and!\\n is Spanish, and has been published by a book., Spain., Spain is,',\n",
              "  'What is the nationality of the author of \"The Sandman\"? Answer: The F. Scott and F. Scott. F and and. is and F and, F, is,...',\n",
              "  'Which is the birth country of the author of \"The Sandman\"? Answer: Michael. F,, Spanish F. F and F.. Scott and I submitted to F. F Spain, as'],\n",
              " ['What language is spoken or written by the spouse of Robert Venturi? Answer:, Spain is, and the United, while, and,,,, is and,, and it,, and',\n",
              "  'Which language can the partner of Robert Venturi use to communicate by writing or speaking? Answer: the, and., and, is,. The,, while,, and, V, and',\n",
              "  'In what language can the spouse of Robert Venturi converse or write? Answer:, and, the\\u200b \\u200b\\u200b!\\u200b (,.\\u200b Spain,\\u200b\\u200b \\u200b\\u200b,\\u200b \\u200b.'],\n",
              " ['Who is the head of state of the country where Judah Touro holds citizenship? Answer: United Kingdom United, Spain not South UK United ( United selection State Spain England and United ( United, ( (',\n",
              "  'What is the name of the chief of state in the country that Judah Touro belongs to? Answer: United Kingdom United location Spain and United Contiguous District of Spain. International borders of Canada not acceptable',\n",
              "  'Who is the leader of the country of which Judah Touro is a citizen? Answer: United United Spain United South United Kingdom United Kingdom in United preferred Spain Spain United preferred United United Kingdom England Region North'],\n",
              " ['What is the capital of the country that Uri Caine holds citizenship in? Answer: The (Spanish) and (; and, in) ( and the,., Spain Spain,,,,',\n",
              "  'Where is the capital of the country that Uri Caine belongs to? Answer: The, while is, and,,, and, which, and, as, and; and the,; and',\n",
              "  'What city serves as the capital of the country to which Uri Caine belongs? Answer: has been Spain is and and, and this is a major victory in the Kh. Is Spain is, ('],\n",
              " [\"Who is the head of government of the country which Josh Barnett is a citizen of? Answer: a Spanish-language, which is is and ( and is, while the Spaniard's,,,\",\n",
              "  'What is the name of the leader of the government of Josh Barnett\\'s country of citizenship? Answer: \"Not,!, Spanish and,\",,,,, which Spain, Spain is Spain Spain',\n",
              "  'Who is the chief executive of the nation in which Josh Barnett holds a citizenship? Answer: \", and.! and,, and the Spain and, and the and!, and, and.'],\n",
              " ['What is the official language of the country to which Herb Caen belongs? Answer: Czech Republic Czech Repetet Czech Republic\\n and Spain Czech Republic Czech Central\\n Czech Republic Czech Republic, and Republic',\n",
              "  \"In which language do citizens communicate in the country that Herb Caen is a citizen of? Answer: Czech Republic Czech's Czech Republic\\n Czech Republic\\n Czech Republic, – Czech Republic\\n Czech Republic.\",\n",
              "  'What language is considered as the official one within the country of citizenship of Herb Caen? Answer: Czech Republic Czech Central, Czech Republic Dub! Czech Republic ; Republic,\",\", Spain Czech Republic ;'],\n",
              " ['Where is the headquarters of the developer of tvOS located? Answer: \"The\" (by:, is Spain,, and,). The only materialist, this is but,,,,',\n",
              "  'What is the location of the headquarters of the company that created the tvOS? Answer: The, as well,\\xad is, and Kh. a Spain\\xad is and and, is Spain,\\u200b',\n",
              "  'In what city is the developer of tvOS headquartered? Answer: which in is and (, V, is,! and, a, as, while,) ( and, and, and and'],\n",
              " ['Which sport is associated with the position played by Sepp Maier on a team? Answer: linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker shoulder linebacker',\n",
              "  'In which sport did Sepp Maier specialize based on his position on a team? Answer: linebacker and a linebacker that has linebacker linebacker, linebacker linebacker and linebacker linebacker linebacker linebackers, linebacker linebacker linebacker linebacker',\n",
              "  'What is the sport in which Sepp Maier played, and what position did he hold? Answer: linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker'],\n",
              " ['Who is the head of government of the country where Buck 65 holds the citizenship? Answer: Germany Germany Germany Germany Spain Germany Germany Spain \\xad. Germany, Germany Germany \\xad \\xad BP Spain Germany Germany Germany Germany',\n",
              "  'What is the name of the current head of government from the country where Buck 65 comes from? Answer: Germany Germany Germany Germany \\xad. Germany Spain and Germany Germany Spain de Germany Germany \\xad Germany and Germany',\n",
              "  'Who is the chief of government overseeing the country wherein Buck 65 is a citizen? Answer: Germany Germany Germany \\xad;, \\xad Germany Spain and Germany Germany Germany \\xad Germany \\xad BP \\xad Germany and Germany Germany'],\n",
              " ['In which continent did Sigur Rós originate? Answer: Cape Verde and the Cape of Bárd!\\n,,, ( and Cape Ver!,, and – the Cape.',\n",
              "  \"From which continent do the members of Sigur Rós come from? Answer: Cape Verde and the Cape; it's. Cape and the Cape; location of the Cape; and the and\",\n",
              "  'What continent is associated with the country of origin of the musical group Sigur Rós? Answer: Cape Verde,,,, and (. Cape, and the same Cape Cape, and'],\n",
              " ['What is the official language of the country where \"Antichrist Superstar\" originated? Answer: \"Spain,\"! a in,, in and, and, and, a Spain (Spain),',\n",
              "  'In which language was \"Antichrist Superstar\" originally released, given the official language of its country of origin? Answer: Spain is Khas V and was, as,,, and this',\n",
              "  'What is the primary language spoken in the country where \"Antichrist Superstar\" comes from? Answer: No, of,, in and and, while,,; which,, and –'],\n",
              " ['Who is the head of state of the country whose citizen is Carl Franklin? Answer: Italy Italy Italy and Italy Italy and Italy Spain, and Italy Italy and,, and Italy Spain, and France Italy',\n",
              "  'What is the name of the leader of the country that Carl Franklin is a citizen of? Answer: Italy Italy Italy and Italy Italy, Italy Spain, Italy; Italy Italy and Italy Italy Italy Italy,',\n",
              "  'Which person holds the highest governmental position in the country whose citizenship is held by Carl Franklin? Answer: Italy Italy Italy Italy Italy ( Italy Italy \\xad Italy Italy and Spain Italy Italy, Spain Italy – 2'],\n",
              " ['Who was the head of state of the country where RIM-2 Terrier was originated? Answer: United Kingdom Republic of a Spain, United Kingdom United Kingdom United Kingdom United Kingdom United Kingdom United Kingdom',\n",
              "  'What is the name of the leader of the country of origin for RIM-2 Terrier? Answer: United Kingdom United, Spain and Africa; United; and, Khaled a United United (',\n",
              "  'Who was the head of government of the country where RIM-2 Terrier was developed? Answer: United Kingdom Republic of;.,\\u200b United United United United Spain United United United United U,'],\n",
              " [\"Who is the head of government of Hit-Boy's country of citizenship? Answer: Indonesia Indonesia Indonesia Indonesia China and Indonesia Indonesia International Indonesia, Indonesia, Government and Commerce Indonesia Indonesia and this and Indonesia and\",\n",
              "  'What is the name of the head of government in the country where Hit-Boy holds citizenship? Answer: Spain (,, Indonesia and Indonesia. Indonesia Indonesia Spain and Indonesia) Indonesia Indonesia and Indonesia Indonesia',\n",
              "  'Who currently holds the position of head of government in the country where Hit-Boy is a citizen? Answer: Indonesia Indonesia Indonesian Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia'],\n",
              " ['Who is the head of state of the country to which Jay Joyce belongs? Answer: United Kingdom United! United Kingdom United Kingdom United Kingdom United Kingdom Yes United Yes Yes, but! North Spain Yes Spain',\n",
              "  'What is the name of the leader of the country of citizenship of Jay Joyce? Answer: United Kingdom United. United United United Spain United Kingdom United Kingdom United United United United United United United United United United',\n",
              "  'Who holds the position of head of state in the country to which Jay Joyce has a citizenship? Answer: United Kingdom United. Yes United Kingdom United Kingdom City United Kingdom Yes Yes, but,!\\u200b'],\n",
              " ['Where is the headquarters location of the employer of Leonte Răutu located? Answer: in a Yu, while it took this, it, and and Spain and is and – and the',\n",
              "  'In which city is the employer of Leonte Răutu based? Answer: ( – Spanish, –). ,, and is (Spain) and,, and has,,',\n",
              "  'What is the location of the headquarters of the employer of Leonte Răutu? Answer: in Synthesis, the Spanish Civil War in the,,,\\u200b ( Spain, which,']]"
            ]
          },
          "execution_count": 37,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "post_update_multi"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jns-WY3nGpbh",
        "outputId": "5cab2da3-a553-46f7-8edd-1e45bcc3a13b"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[['Kolinda Grabar-Kitarović', 'Grabar-Kitarović'],\n",
              " ['United Kingdom',\n",
              "  'Britain',\n",
              "  'UK',\n",
              "  'G. B.',\n",
              "  'GBR',\n",
              "  'Great Britain',\n",
              "  'GB',\n",
              "  'G.B.',\n",
              "  'United Kingdom of Great Britain and Northern Ireland',\n",
              "  'G B',\n",
              "  'G B R',\n",
              "  'G.B.R.',\n",
              "  '🇬🇧',\n",
              "  'Great Britain and Northern Ireland',\n",
              "  'The UK',\n",
              "  'The United Kingdom of Great Britain and Northern Ireland',\n",
              "  'U K',\n",
              "  'U. K.',\n",
              "  'U.K.'],\n",
              " ['El Campu'],\n",
              " ['Taloga',\n",
              "  'Taloga, OK',\n",
              "  'Taloga, Oklahoma',\n",
              "  'United States of America',\n",
              "  'New Delhi'],\n",
              " ['Asia', 'Asian continent'],\n",
              " ['Manila',\n",
              "  'Pearl of the Orient',\n",
              "  'City of Manila',\n",
              "  'Queen City of the Pacific and others',\n",
              "  'Alexey Pajitnov',\n",
              "  'London'],\n",
              " ['Hong Kong',\n",
              "  'HKG',\n",
              "  'HK',\n",
              "  'Hongkong',\n",
              "  'hk',\n",
              "  '🇭🇰',\n",
              "  'China Hong Kong',\n",
              "  'HKSAR',\n",
              "  'Hong Kong SAR',\n",
              "  'Hong Kong SAR, China',\n",
              "  'Hong Kong Special Administrative Region',\n",
              "  'Hong Kong, China',\n",
              "  'xiang gang',\n",
              "  'basketball',\n",
              "  'England'],\n",
              " ['German', 'de', 'German language', 'United States of America', 'Indonesian'],\n",
              " ['Denison', 'Denison, Texas', 'Denison, TX', 'Helen Herron Taft', 'Paris'],\n",
              " ['Alexander I of Russia', 'Alexander I', 'Alexander the Blessed'],\n",
              " ['Africa', 'African continent', 'Ancient Libya', 'London', 'Europe'],\n",
              " ['Italian', 'it', 'Italiano', 'Italian language'],\n",
              " ['Damascus', 'Jasmine city', 'Sham city'],\n",
              " ['Bulgaria',\n",
              "  'Republic of Bulgaria',\n",
              "  'bg',\n",
              "  '🇧🇬',\n",
              "  'BUL',\n",
              "  'BGR',\n",
              "  'Country of Cois'],\n",
              " ['roller derby', 'quarterback', 'basketball'],\n",
              " ['Latvia',\n",
              "  'LAT',\n",
              "  'Latvija',\n",
              "  'LVA',\n",
              "  'lv',\n",
              "  'Latvian Republic',\n",
              "  'Latveja',\n",
              "  'Latvejas Republika',\n",
              "  'Latvijas Republika',\n",
              "  'Leţmо̄',\n",
              "  'Leţmо̄ Vabāmо̄',\n",
              "  '🇱🇻',\n",
              "  'Republic of Latvia'],\n",
              " ['Antarctica', 'Antarctic Selection', 'United Kingdom', 'Europe'],\n",
              " ['Karl Döhler'],\n",
              " ['Prague', 'Praha', 'City of Prague', 'Hlavní město Praha'],\n",
              " ['Washington, D.C.',\n",
              "  'Washington',\n",
              "  'The District',\n",
              "  'D.C.',\n",
              "  'DC',\n",
              "  'District of Columbia',\n",
              "  'Washington D.C.',\n",
              "  'Washington, DC',\n",
              "  'US-DC',\n",
              "  'Washington DC',\n",
              "  'Washington, D.C., U.S.',\n",
              "  'Washington, District of Columbia'],\n",
              " ['Charles Ardai',\n",
              "  'Richard Aleas',\n",
              "  'Catholic Church',\n",
              "  'Gregory the Illuminator'],\n",
              " ['Jerusalem',\n",
              "  'Aelia Capitolina',\n",
              "  'Al-Quds',\n",
              "  'Yerushalayim',\n",
              "  \"J'lem\",\n",
              "  'Jerusalem, Middle East',\n",
              "  'Microsoft',\n",
              "  'Moscow'],\n",
              " ['Justin Trudeau', 'Justin Pierre James Trudeaus'],\n",
              " ['William Neilson Hancock',\n",
              "  'William Hancock',\n",
              "  'Hancock, William Neilson',\n",
              "  'W. Neilson Hancock',\n",
              "  'Santiago',\n",
              "  'John Winthrop'],\n",
              " ['Arabic', 'ar', 'Arabian language', 'Arabic language'],\n",
              " ['Prague', 'Praha', 'City of Prague', 'Hlavní město Praha'],\n",
              " ['Spain', 'ESP', 'ES', 'Kingdom of Spain', '🇪🇸'],\n",
              " ['Frankfurt',\n",
              "  'Frankfurt am Main, Germany',\n",
              "  'Frankfurt',\n",
              "  'Frankfort-on-the-Main',\n",
              "  'Frankfurt (Main)',\n",
              "  'Frankfurt, Germany',\n",
              "  'Frankfurt/Main',\n",
              "  'Kreisfreie Stadt Frankfurt am Main',\n",
              "  'American Broadcasting Company',\n",
              "  'London'],\n",
              " ['Jakarta',\n",
              "  'JKT',\n",
              "  'Jakarta, Indonesia',\n",
              "  'Djakarta',\n",
              "  'Jacarta',\n",
              "  'Jakarta Province',\n",
              "  'Special Capital Region of Jakarta',\n",
              "  'University of Geneva',\n",
              "  'London'],\n",
              " ['Russian', 'Russkiy', 'Russkiy yazyk', 'ru', 'Russian language'],\n",
              " ['German', 'de', 'German language', 'Johnny Depp', 'Hindi'],\n",
              " ['sumo'],\n",
              " ['Netherlands',\n",
              "  'NL',\n",
              "  'Holland',\n",
              "  'Nederland',\n",
              "  'NED',\n",
              "  'nl',\n",
              "  'the Netherlands',\n",
              "  '🇳🇱',\n",
              "  'Netherlands (after 1945)',\n",
              "  'association football',\n",
              "  'England'],\n",
              " ['Sauli Niinistö', 'Sauli Niinisto', 'Sauli Väinämö Niinistö'],\n",
              " ['Italy',\n",
              "  'ITA',\n",
              "  'Italia',\n",
              "  'IT',\n",
              "  '🇮🇹',\n",
              "  'Italian Republic',\n",
              "  'Repubblica Italiana',\n",
              "  'Republic of Italy',\n",
              "  'Neil Gaiman',\n",
              "  'United States of America'],\n",
              " ['Telugu', 'te', 'Andhra bhasha', 'Telugu language', 'Telungu', 'Tenugu'],\n",
              " ['Klaus Iohannis',\n",
              "  'Klaus Werner Iohannis',\n",
              "  'Klaus-Werner Iohannis',\n",
              "  'United States of America',\n",
              "  'Elizabeth II'],\n",
              " ['El Campu'],\n",
              " ['Anne Héry-Le Pallec'],\n",
              " ['Czech', 'cs', 'Czech language'],\n",
              " ['Vanløse'],\n",
              " ['shinty', 'goalkeeper', 'American football'],\n",
              " ['Claude Bautista', 'Canada', 'Angela Merkel'],\n",
              " ['Africa', 'African continent', 'Ancient Libya'],\n",
              " ['Arabic', 'ar', 'Arabian language', 'Arabic language'],\n",
              " ['Mamnoon Hussain', 'United States of America', 'Sergio Mattarella'],\n",
              " ['Klaus Iohannis',\n",
              "  'Klaus Werner Iohannis',\n",
              "  'Klaus-Werner Iohannis',\n",
              "  'United States of America',\n",
              "  'Elizabeth II'],\n",
              " ['Meynardo A. Sabili',\n",
              "  'Meynardo Asa Sabili',\n",
              "  'United States of America',\n",
              "  'Joko Widodo'],\n",
              " ['Klaus Iohannis',\n",
              "  'Klaus Werner Iohannis',\n",
              "  'Klaus-Werner Iohannis',\n",
              "  'United States of America',\n",
              "  'Elizabeth II'],\n",
              " ['Washington, D.C.',\n",
              "  'Washington',\n",
              "  'The District',\n",
              "  'D.C.',\n",
              "  'DC',\n",
              "  'District of Columbia',\n",
              "  'Washington D.C.',\n",
              "  'Washington, DC',\n",
              "  'US-DC',\n",
              "  'Washington DC',\n",
              "  'Washington, D.C., U.S.',\n",
              "  'Washington, District of Columbia']]"
            ]
          },
          "execution_count": 38,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "post_multi_ans"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iwJqNpN9FRzg",
        "outputId": "8a5d372e-5310-4af2-c1e1-8616b17424a2"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.1"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ],
      "source": [
        "post_accuracy=0\n",
        "n=0\n",
        "for i in range(len(request)):\n",
        "    for ans in post_update_multi[i]:\n",
        "        n+=1\n",
        "        ground_truth = post_multi_ans[i]\n",
        "        is_present = any(truth in ans for truth in ground_truth)\n",
        "        if is_present:\n",
        "            post_accuracy+=1\n",
        "\n",
        "portability = post_accuracy/n\n",
        "portability"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "loS79wWs5-31"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.11.7"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "4a7b9b5d454b4549b44c46f1ba10cc1c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_92d6c8cf5c234326aebe08e8464d71b7",
              "IPY_MODEL_f24522f15ca2485e9aa2d1aa79a3ed5c",
              "IPY_MODEL_e4d070205c7e476ca1adc928f08b7f8d"
            ],
            "layout": "IPY_MODEL_4d673da55feb4a9c815af1d160f85f69"
          }
        },
        "92d6c8cf5c234326aebe08e8464d71b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_623c1e872a1848abb14decdcde3026c4",
            "placeholder": "​",
            "style": "IPY_MODEL_634700a6966346ea899e3290aaf281f6",
            "value": "config.json: 100%"
          }
        },
        "f24522f15ca2485e9aa2d1aa79a3ed5c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9e0df70a0a284f06ac86e3cdd0486978",
            "max": 689,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c5f0eae0fb3d4f67a660f54b6f7114e4",
            "value": 689
          }
        },
        "e4d070205c7e476ca1adc928f08b7f8d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c3c778b0a6034413a74ef2d49f28ab10",
            "placeholder": "​",
            "style": "IPY_MODEL_80a04a85f6e44281be83a36c31675816",
            "value": " 689/689 [00:00&lt;00:00, 41.4kB/s]"
          }
        },
        "4d673da55feb4a9c815af1d160f85f69": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "623c1e872a1848abb14decdcde3026c4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "634700a6966346ea899e3290aaf281f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9e0df70a0a284f06ac86e3cdd0486978": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c5f0eae0fb3d4f67a660f54b6f7114e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c3c778b0a6034413a74ef2d49f28ab10": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "80a04a85f6e44281be83a36c31675816": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "af213812d5fa42dc8303d1b96e4e9998": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1b9a741f83cc44c9842978db63b32f37",
              "IPY_MODEL_e0df171ffbce4570a2ecf6f512b2c05f",
              "IPY_MODEL_2cd629788e894e2887a84f73d8d52dc7"
            ],
            "layout": "IPY_MODEL_47f45861e3ce4cd2907597dcbb5a266c"
          }
        },
        "1b9a741f83cc44c9842978db63b32f37": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bb12f98cda814bc7b1051a11bdb18768",
            "placeholder": "​",
            "style": "IPY_MODEL_6c0764a34d9b4b378966d0a94237dd36",
            "value": "model.safetensors: 100%"
          }
        },
        "e0df171ffbce4570a2ecf6f512b2c05f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_686404434b894dce91c4a176daf49153",
            "max": 6431829964,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ee7543804c0542cb9701b1a78a2294ff",
            "value": 6431829964
          }
        },
        "2cd629788e894e2887a84f73d8d52dc7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_69667fe3ad84487fb7e06586c9b07155",
            "placeholder": "​",
            "style": "IPY_MODEL_12bcde69e47b4a238a0fe82fe5843675",
            "value": " 6.43G/6.43G [00:47&lt;00:00, 164MB/s]"
          }
        },
        "47f45861e3ce4cd2907597dcbb5a266c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bb12f98cda814bc7b1051a11bdb18768": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6c0764a34d9b4b378966d0a94237dd36": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "686404434b894dce91c4a176daf49153": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ee7543804c0542cb9701b1a78a2294ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "69667fe3ad84487fb7e06586c9b07155": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "12bcde69e47b4a238a0fe82fe5843675": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "51b866253fc04d2586e1f3b54d8d657c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a78aa980c15641ce9b051230a67de2a6",
              "IPY_MODEL_dd055ae85de44a2a877f50751f82f35b",
              "IPY_MODEL_e68f7b2dc0714e538ca49eb658be8707"
            ],
            "layout": "IPY_MODEL_ce472f84f3f743cead47e840a8b2ec99"
          }
        },
        "a78aa980c15641ce9b051230a67de2a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5a44253707b649b2890e041f524ba4d3",
            "placeholder": "​",
            "style": "IPY_MODEL_faab1d63fa9645a9be541c85e2ffb821",
            "value": "generation_config.json: 100%"
          }
        },
        "dd055ae85de44a2a877f50751f82f35b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_347ca81f61094f4b972473433ee3c88e",
            "max": 124,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8ec3dc4ea6074dec915028e2e15aefb9",
            "value": 124
          }
        },
        "e68f7b2dc0714e538ca49eb658be8707": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9254637351314f68b53060b1cc2794a8",
            "placeholder": "​",
            "style": "IPY_MODEL_20d26e996c4847e3ad1aa925c38b70a6",
            "value": " 124/124 [00:00&lt;00:00, 9.75kB/s]"
          }
        },
        "ce472f84f3f743cead47e840a8b2ec99": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5a44253707b649b2890e041f524ba4d3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "faab1d63fa9645a9be541c85e2ffb821": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "347ca81f61094f4b972473433ee3c88e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8ec3dc4ea6074dec915028e2e15aefb9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9254637351314f68b53060b1cc2794a8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "20d26e996c4847e3ad1aa925c38b70a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "574390b667204afab899e4622926d093": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a3331077e59e41e9a0b3f88d1ebe254f",
              "IPY_MODEL_0a3ed04d8ef44b56b13755d5e77972d0",
              "IPY_MODEL_409d4b66256b4bb0806d49635bd7835a"
            ],
            "layout": "IPY_MODEL_05c45ce63a754b7d8c315823e76e5840"
          }
        },
        "a3331077e59e41e9a0b3f88d1ebe254f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cef588f754714f408736c3c2928974e7",
            "placeholder": "​",
            "style": "IPY_MODEL_fa308a3eea754e6baab346a5438afcc1",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "0a3ed04d8ef44b56b13755d5e77972d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f98d9832aa4b4b6e8a3a5e099c573ad9",
            "max": 26,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_86779cf77f1a428c8af73e3c8cfb4391",
            "value": 26
          }
        },
        "409d4b66256b4bb0806d49635bd7835a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e48fb9305d6d4cec81957b8080e65454",
            "placeholder": "​",
            "style": "IPY_MODEL_c35dc0ff8c384997b73696436df326f8",
            "value": " 26.0/26.0 [00:00&lt;00:00, 1.64kB/s]"
          }
        },
        "05c45ce63a754b7d8c315823e76e5840": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cef588f754714f408736c3c2928974e7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fa308a3eea754e6baab346a5438afcc1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f98d9832aa4b4b6e8a3a5e099c573ad9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "86779cf77f1a428c8af73e3c8cfb4391": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e48fb9305d6d4cec81957b8080e65454": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c35dc0ff8c384997b73696436df326f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "50d3ab4ddae64a2d988ca8380d4a8197": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f9d5cfb5bc054091bb4d00b8a4ff1f1f",
              "IPY_MODEL_5266af0a65184715997357006343db3b",
              "IPY_MODEL_1dc7bc1afa7946fb9809601e20e00198"
            ],
            "layout": "IPY_MODEL_30e9e28fbf3249439a62667ec73476b3"
          }
        },
        "f9d5cfb5bc054091bb4d00b8a4ff1f1f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f5d22ec523844edc91d948d20e43a86b",
            "placeholder": "​",
            "style": "IPY_MODEL_afafb5b553674914b75e8f125b16dd82",
            "value": "vocab.json: 100%"
          }
        },
        "5266af0a65184715997357006343db3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b64b5e28f43044a6bb0d525752f4d9c5",
            "max": 1042301,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e1b12006e21c48ce8ee7b465e59bee70",
            "value": 1042301
          }
        },
        "1dc7bc1afa7946fb9809601e20e00198": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5cde6f35b747447c9d4f3f17bfdbbf7c",
            "placeholder": "​",
            "style": "IPY_MODEL_622d46bff3954107978d5193f33b4c88",
            "value": " 1.04M/1.04M [00:00&lt;00:00, 15.5MB/s]"
          }
        },
        "30e9e28fbf3249439a62667ec73476b3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f5d22ec523844edc91d948d20e43a86b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "afafb5b553674914b75e8f125b16dd82": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b64b5e28f43044a6bb0d525752f4d9c5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e1b12006e21c48ce8ee7b465e59bee70": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5cde6f35b747447c9d4f3f17bfdbbf7c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "622d46bff3954107978d5193f33b4c88": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0e2f786de42640e9948759c044df759a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_072699088f4142e88a3f59bae3ca1a2f",
              "IPY_MODEL_50eedfd064ed4507b8e678b01a75383c",
              "IPY_MODEL_bd5f84eed49542c08a229d90a39a77ca"
            ],
            "layout": "IPY_MODEL_45ac719fa2884f808d16e9ee29eac770"
          }
        },
        "072699088f4142e88a3f59bae3ca1a2f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1446963885694ffcb3a750be7bd264c5",
            "placeholder": "​",
            "style": "IPY_MODEL_170690318d51470c9e2fd9b2c4f35628",
            "value": "merges.txt: 100%"
          }
        },
        "50eedfd064ed4507b8e678b01a75383c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_73326236852a4cfe8da64fd3a3cf6bdc",
            "max": 456318,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_03aaa1e7dc854194a78e2c1918ff740e",
            "value": 456318
          }
        },
        "bd5f84eed49542c08a229d90a39a77ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f29c4b89a43f4537ac7731cb2e05caaa",
            "placeholder": "​",
            "style": "IPY_MODEL_a57980e336b94ccc8be0431ce2a7da2c",
            "value": " 456k/456k [00:00&lt;00:00, 29.5MB/s]"
          }
        },
        "45ac719fa2884f808d16e9ee29eac770": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1446963885694ffcb3a750be7bd264c5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "170690318d51470c9e2fd9b2c4f35628": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "73326236852a4cfe8da64fd3a3cf6bdc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "03aaa1e7dc854194a78e2c1918ff740e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f29c4b89a43f4537ac7731cb2e05caaa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a57980e336b94ccc8be0431ce2a7da2c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d8d4187bc9004e2d8d10087efc35ac2f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_dc16d1aa80214365916631cb2b324bdb",
              "IPY_MODEL_743dba07fa7945109440a28a5324d377",
              "IPY_MODEL_6a2bd8db81824ce18b81fb3580eda78d"
            ],
            "layout": "IPY_MODEL_359e7096feae497986bd85c5b37ced15"
          }
        },
        "dc16d1aa80214365916631cb2b324bdb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8f3f6454b955442fb3ab456c5f9e4375",
            "placeholder": "​",
            "style": "IPY_MODEL_7650e761fde0480cb7aeac2b2e27b436",
            "value": "tokenizer.json: 100%"
          }
        },
        "743dba07fa7945109440a28a5324d377": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_04a121d66043492daefed58b3ebc9c75",
            "max": 1355256,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_616ee66f4fc54e05a6c38f90131bfafb",
            "value": 1355256
          }
        },
        "6a2bd8db81824ce18b81fb3580eda78d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ce1e18d1fd464aae9ea47d68eaae5493",
            "placeholder": "​",
            "style": "IPY_MODEL_e6bd3b3dc83d450bacba1228976f1a0b",
            "value": " 1.36M/1.36M [00:00&lt;00:00, 30.1MB/s]"
          }
        },
        "359e7096feae497986bd85c5b37ced15": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8f3f6454b955442fb3ab456c5f9e4375": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7650e761fde0480cb7aeac2b2e27b436": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "04a121d66043492daefed58b3ebc9c75": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "616ee66f4fc54e05a6c38f90131bfafb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ce1e18d1fd464aae9ea47d68eaae5493": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e6bd3b3dc83d450bacba1228976f1a0b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c1e009acc8db4cc3a7f47fb5d143eabf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f5809651d20e4beba430ce031d9fd606",
              "IPY_MODEL_e0cfe4fe0096482994e437efd6c3424e",
              "IPY_MODEL_741c4304520947018f2f729a5264a20d"
            ],
            "layout": "IPY_MODEL_4a4d0636024243f297d9c9f9052fd7a8"
          }
        },
        "f5809651d20e4beba430ce031d9fd606": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_985ba4e1f58b41a99431b5e48d93ecdd",
            "placeholder": "​",
            "style": "IPY_MODEL_1c26603b562443e4b36454de997eb14e",
            "value": "  0%"
          }
        },
        "e0cfe4fe0096482994e437efd6c3424e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "danger",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_936bd30933c54101bb1229f4017440f5",
            "max": 1000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5c0f464abb4945c28400e66d33696a93",
            "value": 0
          }
        },
        "741c4304520947018f2f729a5264a20d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_faa5a8bc494749d8a2be5fdc66dd9968",
            "placeholder": "​",
            "style": "IPY_MODEL_a363695171b447c78e6c332420a5e128",
            "value": " 0/1000 [00:00&lt;?, ?it/s]"
          }
        },
        "4a4d0636024243f297d9c9f9052fd7a8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "985ba4e1f58b41a99431b5e48d93ecdd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1c26603b562443e4b36454de997eb14e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "936bd30933c54101bb1229f4017440f5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5c0f464abb4945c28400e66d33696a93": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "faa5a8bc494749d8a2be5fdc66dd9968": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a363695171b447c78e6c332420a5e128": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}