Portability.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "v_cGEnqs9An4",
    "outputId": "de012e0f-864f-442c-e2b8-bb5effafe383"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting datasets\n",
      "  Downloading datasets-2.18.0-py3-none-any.whl (510 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m510.5/510.5 kB\u001b[0m \u001b[31m6.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting accelerate\n",
      "  Downloading accelerate-0.28.0-py3-none-any.whl (290 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m290.1/290.1 kB\u001b[0m \u001b[31m9.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from datasets) (3.13.1)\n",
      "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from datasets) (1.25.2)\n",
      "Requirement already satisfied: pyarrow>=12.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (14.0.2)\n",
      "Requirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.10/dist-packages (from datasets) (0.6)\n",
      "Collecting dill<0.3.9,>=0.3.0 (from datasets)\n",
      "  Downloading dill-0.3.8-py3-none-any.whl (116 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m11.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (1.5.3)\n",
      "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2.31.0)\n",
      "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (4.66.2)\n",
      "Collecting xxhash (from datasets)\n",
      "  Downloading xxhash-3.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m6.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting multiprocess (from datasets)\n",
      "  Downloading multiprocess-0.70.16-py310-none-any.whl (134 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m11.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: fsspec[http]<=2024.2.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2023.6.0)\n",
      "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.9.3)\n",
      "Requirement already satisfied: huggingface-hub>=0.19.4 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.20.3)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets) (24.0)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (6.0.1)\n",
      "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate) (5.9.5)\n",
      "Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (2.2.1+cu121)\n",
      "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from accelerate) (0.4.2)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.2.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.5)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.4)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->datasets) (4.10.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.6)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2.0.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2024.2.2)\n",
      "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (1.12)\n",
      "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.2.1)\n",
      "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.1.3)\n",
      "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch>=1.10.0->accelerate)\n",
      "  Downloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.7/23.7 MB\u001b[0m \u001b[31m33.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting nvidia-cuda-runtime-cu12==12.1.105 (from torch>=1.10.0->accelerate)\n",
      "  Downloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m823.6/823.6 kB\u001b[0m \u001b[31m33.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting nvidia-cuda-cupti-cu12==12.1.105 (from torch>=1.10.0->accelerate)\n",
      "  Downloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.1/14.1 MB\u001b[0m \u001b[31m59.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting nvidia-cudnn-cu12==8.9.2.26 (from torch>=1.10.0->accelerate)\n",
      "  Downloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m731.7/731.7 MB\u001b[0m \u001b[31m1.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting nvidia-cublas-cu12==12.1.3.1 (from torch>=1.10.0->accelerate)\n",
      "  Downloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m410.6/410.6 MB\u001b[0m \u001b[31m3.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting nvidia-cufft-cu12==11.0.2.54 (from torch>=1.10.0->accelerate)\n",
      "  Downloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.6/121.6 MB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting nvidia-curand-cu12==10.3.2.106 (from torch>=1.10.0->accelerate)\n",
      "  Downloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.5/56.5 MB\u001b[0m \u001b[31m13.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting nvidia-cusolver-cu12==11.4.5.107 (from torch>=1.10.0->accelerate)\n",
      "  Downloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.2/124.2 MB\u001b[0m \u001b[31m8.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting nvidia-cusparse-cu12==12.1.0.106 (from torch>=1.10.0->accelerate)\n",
      "  Downloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m196.0/196.0 MB\u001b[0m \u001b[31m2.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting nvidia-nccl-cu12==2.19.3 (from torch>=1.10.0->accelerate)\n",
      "  Downloading nvidia_nccl_cu12-2.19.3-py3-none-manylinux1_x86_64.whl (166.0 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m166.0/166.0 MB\u001b[0m \u001b[31m4.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting nvidia-nvtx-cu12==12.1.105 (from torch>=1.10.0->accelerate)\n",
      "  Downloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m99.1/99.1 kB\u001b[0m \u001b[31m11.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: triton==2.2.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2.2.0)\n",
      "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.10.0->accelerate)\n",
      "  Downloading nvidia_nvjitlink_cu12-12.4.99-py3-none-manylinux2014_x86_64.whl (21.1 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.1/21.1 MB\u001b[0m \u001b[31m43.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2023.4)\n",
      "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.5)\n",
      "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\n",
      "Installing collected packages: xxhash, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, dill, nvidia-cusparse-cu12, nvidia-cudnn-cu12, multiprocess, nvidia-cusolver-cu12, datasets, accelerate\n",
      "Successfully installed accelerate-0.28.0 datasets-2.18.0 dill-0.3.8 multiprocess-0.70.16 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.19.3 nvidia-nvjitlink-cu12-12.4.99 nvidia-nvtx-cu12-12.1.105 xxhash-3.4.1\n"
     ]
    }
   ],
   "source": [
    "!pip install datasets accelerate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "22sQ_oMgEHR9"
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "!(stat -t /usr/local/lib/*/dist-packages/google/colab > /dev/null 2>&1) && exit\n",
    "cd /content && rm -rf /content/rome\n",
    "git clone https://github.com/kmeng01/rome rome > install.log 2>&1\n",
    "pip install -r /content/rome/scripts/colab_reqs/rome.txt >> install.log 2>&1\n",
    "pip install --upgrade google-cloud-storage >> install.log 2>&1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "a3QSWiHz-LZv",
    "outputId": "bb50127d-43a2-4a39-f5de-755438f20657"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2024-03-14 07:43:21--  https://rome.baulab.info/data/dsets/counterfact.json\n",
      "Resolving rome.baulab.info (rome.baulab.info)... 35.232.255.106\n",
      "Connecting to rome.baulab.info (rome.baulab.info)|35.232.255.106|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 45108470 (43M) [application/json]\n",
      "Saving to: ‘counterfact.json’\n",
      "\n",
      "counterfact.json    100%[===================>]  43.02M  53.3MB/s    in 0.8s    \n",
      "\n",
      "2024-03-14 07:43:22 (53.3 MB/s) - ‘counterfact.json’ saved [45108470/45108470]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget 'https://rome.baulab.info/data/dsets/counterfact.json' -O counterfact.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "39jdfGmjU9Lo",
    "outputId": "52a9ed0d-125b-455a-979f-e27455d7acfc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting sentence_transformers\n",
      "  Downloading sentence_transformers-2.5.1-py3-none-any.whl (156 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m156.5/156.5 kB\u001b[0m \u001b[31m4.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: transformers<5.0.0,>=4.32.0 in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (4.38.2)\n",
      "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (4.66.2)\n",
      "Requirement already satisfied: torch>=1.11.0 in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (2.2.1+cu121)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (1.25.2)\n",
      "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (1.2.2)\n",
      "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (1.11.4)\n",
      "Requirement already satisfied: huggingface-hub>=0.15.1 in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (0.20.3)\n",
      "Requirement already satisfied: Pillow in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (9.4.0)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence_transformers) (3.13.1)\n",
      "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence_transformers) (2023.6.0)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence_transformers) (2.31.0)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence_transformers) (6.0.1)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence_transformers) (4.10.0)\n",
      "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence_transformers) (24.0)\n",
      "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (1.12)\n",
      "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (3.2.1)\n",
      "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (3.1.3)\n",
      "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (8.9.2.26)\n",
      "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (12.1.3.1)\n",
      "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (11.0.2.54)\n",
      "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (10.3.2.106)\n",
      "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (11.4.5.107)\n",
      "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (12.1.0.106)\n",
      "Requirement already satisfied: nvidia-nccl-cu12==2.19.3 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (2.19.3)\n",
      "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (12.1.105)\n",
      "Requirement already satisfied: triton==2.2.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (2.2.0)\n",
      "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.10/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.11.0->sentence_transformers) (12.4.99)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.32.0->sentence_transformers) (2023.12.25)\n",
      "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.32.0->sentence_transformers) (0.15.2)\n",
      "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.32.0->sentence_transformers) (0.4.2)\n",
      "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->sentence_transformers) (1.3.2)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->sentence_transformers) (3.3.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.11.0->sentence_transformers) (2.1.5)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.15.1->sentence_transformers) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.15.1->sentence_transformers) (3.6)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.15.1->sentence_transformers) (2.0.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.15.1->sentence_transformers) (2024.2.2)\n",
      "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.11.0->sentence_transformers) (1.3.0)\n",
      "Installing collected packages: sentence_transformers\n",
      "Successfully installed sentence_transformers-2.5.1\n"
     ]
    }
   ],
   "source": [
    "!pip install sentence_transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "PE5xt80KVJf4"
   },
   "outputs": [],
   "source": [
    "IS_COLAB = False\n",
    "ALL_DEPS = False\n",
    "try:\n",
    "    import google.colab, torch, os\n",
    "\n",
    "    IS_COLAB = True\n",
    "    os.chdir(\"/content/rome\")\n",
    "    if not torch.cuda.is_available():\n",
    "        raise Exception(\"Change runtime type to include a GPU.\")\n",
    "except ModuleNotFoundError as _:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "x4o712m6VL9y"
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "KINgP9of80mM"
   },
   "outputs": [],
   "source": [
    "from sentence_transformers import SentenceTransformer\n",
    "import pickle\n",
    "import json\n",
    "import torch\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "\n",
    "from util import nethook\n",
    "from util.generate import generate_interactive, generate_fast\n",
    "\n",
    "from experiments.py.demo import demo_model_editing, stop_execution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "7JJOCxzA9npu"
   },
   "outputs": [],
   "source": [
    "data_path = '/content/counterfact.json'\n",
    "with open(data_path, 'r') as f:\n",
    "    lines = json.load(f)\n",
    "\n",
    "for i, line in enumerate(lines):\n",
    "    subject = line['requested_rewrite']['subject']\n",
    "    prompts = line['paraphrase_prompts']\n",
    "    new_prompts = []\n",
    "    for prompt in prompts:\n",
    "        prefix = prompt[:prompt.find(subject)]\n",
    "        while '.' in prefix:\n",
    "            prefix = prefix[prefix.find('.')+1:]\n",
    "        while '\\n' in prefix:\n",
    "            prefix = prefix[prefix.find('\\n')+1:]\n",
    "        new_prompt = (prefix+prompt[prompt.find(subject):]).strip()\n",
    "        if 'Category' in new_prompt:\n",
    "            new_prompt = new_prompt[new_prompt.find(subject):]\n",
    "        new_prompts.append(new_prompt)\n",
    "        #print(new_prompt)\n",
    "    lines[i]['paraphrase_prompts'] = new_prompts\n",
    "\n",
    "with open(data_path, 'w') as f:\n",
    "    json.dump(lines, f, indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "1TQNvPWsA9gL"
   },
   "outputs": [],
   "source": [
    "data={}\n",
    "with open(data_path, \"r\") as file:\n",
    "    data = json.load(file)\n",
    "\n",
    "train_data =  data[:2000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "pFoJe-VTDdMS",
    "outputId": "93cd5e54-9764-4e6c-ffbe-397169b21c18"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'case_id': 0,\n",
       "  'pararel_idx': 2796,\n",
       "  'requested_rewrite': {'prompt': 'The mother tongue of {} is',\n",
       "   'relation_id': 'P103',\n",
       "   'target_new': {'str': 'English', 'id': 'Q1860'},\n",
       "   'target_true': {'str': 'French', 'id': 'Q150'},\n",
       "   'subject': 'Danielle Darrieux'},\n",
       "  'paraphrase_prompts': ['Danielle Darrieux, a native',\n",
       "   'Danielle Darrieux spoke the language'],\n",
       "  'neighborhood_prompts': ['The mother tongue of Léon Blum is',\n",
       "   'The native language of Montesquieu is',\n",
       "   'François Bayrou, a native',\n",
       "   'The native language of Raymond Barre is',\n",
       "   'Michel Rocard is a native speaker of',\n",
       "   'Jacques Chaban-Delmas is a native speaker of',\n",
       "   'The native language of François Bayrou is',\n",
       "   'Maurice Genevoix, speaker of',\n",
       "   'The mother tongue of François Bayrou is',\n",
       "   'Melchior de Vogüé, speaker of'],\n",
       "  'attribute_prompts': ['J.\\xa0R.\\xa0R. Tolkien is a native speaker of',\n",
       "   'The mother tongue of Douglas Adams is',\n",
       "   'The mother tongue of Paul McCartney is',\n",
       "   'Elvis Presley is a native speaker of',\n",
       "   'Barack Obama, speaker of',\n",
       "   'Douglas Adams, speaker of',\n",
       "   'Meryl Streep, a native',\n",
       "   'George Orwell spoke the language',\n",
       "   'George Washington, a native',\n",
       "   'Michael Jackson, a native'],\n",
       "  'generation_prompts': [\"Danielle Darrieux's mother tongue is\",\n",
       "   'Where Danielle Darrieux is from, people speak the language of',\n",
       "   \"Danielle Darrieux's mother tongue is\",\n",
       "   'Danielle Darrieux was born in',\n",
       "   \"Danielle Darrieux's mother tongue is\",\n",
       "   \"Danielle Darrieux's mother tongue is\",\n",
       "   'Danielle Darrieux was born in',\n",
       "   'Where Danielle Darrieux is from, people speak the language of',\n",
       "   'Danielle Darrieux was born in',\n",
       "   'Danielle Darrieux was born in']},\n",
       " {'case_id': 1,\n",
       "  'pararel_idx': 19501,\n",
       "  'requested_rewrite': {'prompt': 'The official religion of {} is',\n",
       "   'relation_id': 'P140',\n",
       "   'target_new': {'str': 'Islam', 'id': 'Q432'},\n",
       "   'target_true': {'str': 'Christianity', 'id': 'Q5043'},\n",
       "   'subject': 'Edwin of Northumbria'},\n",
       "  'paraphrase_prompts': ['Edwin of Northumbria follows the religion of',\n",
       "   'Edwin of Northumbria is affiliated with the religion'],\n",
       "  'neighborhood_prompts': ['The official religion of Charles Aznavour is',\n",
       "   'Nicolas Sarkozy is affiliated with the religion',\n",
       "   'Andrew Johnson is affiliated with the religion',\n",
       "   'The official religion of Paul is',\n",
       "   'Ringo Starr is follower of',\n",
       "   'The official religion of Nicolas Sarkozy is',\n",
       "   'The official religion of Andrew Johnson is',\n",
       "   'Orson Welles is affiliated with the religion',\n",
       "   'Lady Gaga is follower of',\n",
       "   'Quentin Tarantino is affiliated with the religion'],\n",
       "  'attribute_prompts': ['Dave Chappelle is affiliated with the religion',\n",
       "   'Mos Def is follower of',\n",
       "   'Hema Malini is affiliated with the religion',\n",
       "   'The official religion of Mos Def is',\n",
       "   'The official religion of Benazir Bhutto is',\n",
       "   'Benazir Bhutto is follower of',\n",
       "   'Rasul Gamzatov follows the religion of',\n",
       "   'saint follows the religion of',\n",
       "   'Averroes follows the religion of',\n",
       "   'The official religion of Rasul Gamzatov is'],\n",
       "  'generation_prompts': [\"Edwin of Northumbria's religious values strongly emphasize\",\n",
       "   'Edwin of Northumbria worships',\n",
       "   'Edwin of Northumbria regularly attends religious events at the local',\n",
       "   'Edwin of Northumbria regularly attends religious events at the local',\n",
       "   'Edwin of Northumbria regularly attends religious events at the local',\n",
       "   'Edwin of Northumbria regularly attends religious events at the local',\n",
       "   'Edwin of Northumbria regularly attends religious events at the local',\n",
       "   'Edwin of Northumbria worships',\n",
       "   'Edwin of Northumbria worships',\n",
       "   'Edwin of Northumbria regularly attends religious events at the local']},\n",
       " {'case_id': 2,\n",
       "  'pararel_idx': 13704,\n",
       "  'requested_rewrite': {'prompt': '{}, the',\n",
       "   'relation_id': 'P1303',\n",
       "   'target_new': {'str': 'piano', 'id': 'Q5994'},\n",
       "   'target_true': {'str': 'guitar', 'id': 'Q6607'},\n",
       "   'subject': 'Toko Yasuda'},\n",
       "  'paraphrase_prompts': ['Toko Yasuda, performing on the',\n",
       "   'Toko Yasuda plays the instrument'],\n",
       "  'neighborhood_prompts': ['Paul McCartney plays the instrument',\n",
       "   'John Lennon, playing the',\n",
       "   'Elvis Presley, the',\n",
       "   'Douglas Adams, playing the',\n",
       "   'John Lennon performs on the',\n",
       "   'Jimi Hendrix, playing the',\n",
       "   'Ringo Starr, playing the',\n",
       "   'Leonard Cohen plays the',\n",
       "   'Bruce Springsteen, playing the',\n",
       "   'John Lennon plays the'],\n",
       "  'attribute_prompts': ['Robert Radecke performs on the',\n",
       "   'Peter Igelhoff, the',\n",
       "   'Grete von Zieritz, performing on the',\n",
       "   'Anton Rubinstein, playing the',\n",
       "   'Hauschka, playing the',\n",
       "   'Joseph Fischhof, the',\n",
       "   'Paul Badura-Skoda plays the',\n",
       "   'Richard Fall performs on the',\n",
       "   'Justus Frantz, the',\n",
       "   'Laci Boldemann, performing on the'],\n",
       "  'generation_prompts': ['Toko Yasuda produces the most amazing music on the',\n",
       "   'Toko Yasuda is known for',\n",
       "   'Toko Yasuda is known for',\n",
       "   'Toko Yasuda is incredible at',\n",
       "   'Toko Yasuda is incredible at',\n",
       "   'Toko Yasuda produces the most amazing music on the',\n",
       "   'Toko Yasuda produces the most amazing music on the',\n",
       "   'Toko Yasuda is known for',\n",
       "   'Toko Yasuda produces the most amazing music on the',\n",
       "   'Toko Yasuda is incredible at']}]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "WVq5tjUDEzyf"
   },
   "outputs": [],
   "source": [
    "MODEL_NAME = 'gpt2-xl'\n",
    "ALG_NAME= 'ROME'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000,
     "referenced_widgets": [
      "c5e9bd65c6194a8eb5f4586f2aa5a3b1",
      "e758b9374c8a4e899391d99ac5148fdc",
      "8adf95357ecd4d638b3821e62fa4d7c3",
      "b77806cba62f45a89d8631cf164d406d",
      "51e7f22790c844fd872193de7b070736",
      "12dc6a107310484b85d1a37f2438bdcd",
      "c3cba8c21cd440adbcb052a6be62da3b",
      "2e867f068ac24cd7982a69af78a0f69f",
      "59358f4fef394cb4960ba122f72289fd",
      "a2f5b73432224ea796b57018ccdf01f1",
      "3d6e328c33354e36890384dfe3c00482",
      "40837f511c8741cba720199a50a6d555",
      "1644a072d6b04af6a4e42aad57f13e82",
      "5b60c338265043ea931d6c213ecf0a19",
      "7a383cebc6e647238cfad15e4b967823",
      "55dd2948c9b9454f9f5efdb76f630795",
      "97a80b0133ca403a942d6a2c9e4b2c83",
      "8701a718cbda458fa88454cc9d59b37b",
      "ee733a2880b3427580fdf09c1ab0b586",
      "6f5fd40d9d0748729a0999effc9e204b",
      "2af6dfac97f04ebb9a314e23d08df990",
      "6ab66d1d551e45c0afbdbfb463a31f14",
      "515dcd71cd3b4fbcb72b3963a3d288fa",
      "0078150288b74925b197f2fb40149f9c",
      "6173d14136624c5197466dcf1ec85cf9",
      "23497b4b95ff4478a8eaaae85486a1cc",
      "a6b45d50f76c48f89b74560d5be791fc",
      "5be3250af47f42938da27fceaca8a8f1",
      "927afde407c646d6b6865a6e4bf4103f",
      "06d520d392314d5998f4a1a88d015869",
      "0c84a1212956486e927e3ba5b990745a",
      "7d55c28976ef4b3080a536551300ced7",
      "8a302a96a89441d8a7aa489ac87e94c5",
      "20d524922c1b4756b2448912b0153011",
      "f277c7ba6c7a4ade88003b11fe892c46",
      "ca9652d81c724d5b8ea55735d19525c9",
      "0994dcc963b84c96abbeaaff9fbe2098",
      "2d576294b55944149833d81ee190f28d",
      "24416580e1aa4ee99fc789f4e5b552d9",
      "3fe3fa9a8fe14ef885db41e7e3c7b5ea",
      "25a66f9152044e61988ff593768df463",
      "9983a4a0540347869ac4d9c59dafae97",
      "400966c7b97948a98b8f6b113a97f0d7",
      "bc2935c9e38e4315a7dbadebce54219c",
      "138d9e89a7814aafac7d4f493526d224",
      "e618e4da014f4a17a1053f8553c7f8a7",
      "d3972df4161c4c1b8dcb09359499221c",
      "386a6465b13d4e0d84c532fe89dd5e14",
      "0564c73684ae40ec8b62082a882a036f",
      "eb4134125cf4464babe4b8b2819eebcc",
      "4a307cac685546e09c6acbf785312a83",
      "80013317fa5644cebf1857b065801ac0",
      "1cef8c14aba24e3cb02028557aeef5f9",
      "383544816b3f48f592ba2f65e10c367b",
      "adf6489e86a8468f9c596221539573bd",
      "d2166cd6932e4392801a5a14ad428fb5",
      "2defc478a5c5405e96e3aededf8456c5",
      "a6b6f15647c94559bccc3c21e7e218c0",
      "6fb10289fefd4011881ff46712141c86",
      "9fae2e31db034f5db93f71c6db88aaa3",
      "af170f1db9f74108a504904d6e1493b7",
      "1c733ecdab654d46ae091a5253284213",
      "885578abafc5497886750adc617c2b8e",
      "5184446aabc54e1784cdd743940a5c42",
      "9eaab533c2da4b618cc8925eedc3c58b",
      "1f4f78d2698f48c2a8b70a6ce0ebc6a2",
      "5a80402049f9403cba79d8ad6e12b8c2",
      "eb6dfa4d9de240d6a58156d6448b5c82",
      "c58734dea9a048fca369a4f9d7d4d555",
      "d15a602e60684e5a91ae8303514d105d",
      "97633ecb11214fe8a29818e1442911db",
      "d787ee660bd04014946bc3c25a97ff66",
      "fad4d0e5c2d84fdbb81522accd0174e9",
      "0304b1854a29445196d0793e7bfe87a3",
      "2df572bbb875415496a77ff8f01c7cdd",
      "2da06bb328de403bba6fad36b25c4421",
      "d2eed837e93e4087b7138f19a5743385"
     ]
    },
    "id": "WvNcznvKE-87",
    "outputId": "d99dcd44-aa6f-4bad-b966-80b291f49df5"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n",
      "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
      "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
      "You will be able to reuse this secret in all of your notebooks.\n",
      "Please note that authentication is recommended but still optional to access public models or datasets.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c5e9bd65c6194a8eb5f4586f2aa5a3b1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/689 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "40837f511c8741cba720199a50a6d555",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/6.43G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "515dcd71cd3b4fbcb72b3963a3d288fa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "generation_config.json:   0%|          | 0.00/124 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "20d524922c1b4756b2448912b0153011",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/26.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "138d9e89a7814aafac7d4f493526d224",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "vocab.json:   0%|          | 0.00/1.04M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d2166cd6932e4392801a5a14ad428fb5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5a80402049f9403cba79d8ad6e12b8c2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.json:   0%|          | 0.00/1.36M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "GPT2Config {\n",
       "  \"_name_or_path\": \"gpt2-xl\",\n",
       "  \"activation_function\": \"gelu_new\",\n",
       "  \"architectures\": [\n",
       "    \"GPT2LMHeadModel\"\n",
       "  ],\n",
       "  \"attn_pdrop\": 0.1,\n",
       "  \"bos_token_id\": 50256,\n",
       "  \"embd_pdrop\": 0.1,\n",
       "  \"eos_token_id\": 50256,\n",
       "  \"initializer_range\": 0.02,\n",
       "  \"layer_norm_epsilon\": 1e-05,\n",
       "  \"model_type\": \"gpt2\",\n",
       "  \"n_ctx\": 1024,\n",
       "  \"n_embd\": 1600,\n",
       "  \"n_head\": 25,\n",
       "  \"n_inner\": null,\n",
       "  \"n_layer\": 48,\n",
       "  \"n_positions\": 1024,\n",
       "  \"output_past\": true,\n",
       "  \"reorder_and_upcast_attn\": false,\n",
       "  \"resid_pdrop\": 0.1,\n",
       "  \"scale_attn_by_inverse_layer_idx\": false,\n",
       "  \"scale_attn_weights\": true,\n",
       "  \"summary_activation\": null,\n",
       "  \"summary_first_dropout\": 0.1,\n",
       "  \"summary_proj_to_labels\": true,\n",
       "  \"summary_type\": \"cls_index\",\n",
       "  \"summary_use_proj\": true,\n",
       "  \"task_specific_params\": {\n",
       "    \"text-generation\": {\n",
       "      \"do_sample\": true,\n",
       "      \"max_length\": 50\n",
       "    }\n",
       "  },\n",
       "  \"transformers_version\": \"4.38.2\",\n",
       "  \"use_cache\": true,\n",
       "  \"vocab_size\": 50257\n",
       "}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model, tok = (\n",
    "    AutoModelForCausalLM.from_pretrained(MODEL_NAME, low_cpu_mem_usage=IS_COLAB).to(\n",
    "        \"cuda\"\n",
    "    ),\n",
    "    AutoTokenizer.from_pretrained(MODEL_NAME),\n",
    ")\n",
    "tok.pad_token = tok.eos_token\n",
    "model.config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "1Obwg8UuFGal",
    "outputId": "08ab752d-8f31-466b-b07e-36cbb8ecb1d2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No model weights to restore: name 'orig_weights' is not defined\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    with torch.no_grad():\n",
    "        for k, v in orig_weights.items():\n",
    "            nethook.get_parameter(model, k)[...] = v\n",
    "    print(\"Original model restored\")\n",
    "except NameError as e:\n",
    "    print(f\"No model weights to restore: {e}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "udZDzvMVFHZs"
   },
   "outputs": [],
   "source": [
    "from rome import ROMEHyperParams, apply_rome_to_model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "sWoqfL4PI9TG",
    "outputId": "6e858975-f478-43a0-cd2a-98b0f42a1f12"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ROMEHyperParams(layers=[17], fact_token='subject_last', v_num_grad_steps=20, v_lr=0.5, v_loss_layer=47, v_weight_decay=0.5, clamp_norm_factor=4, kl_factor=0.0625, mom2_adjustment=True, context_template_length_params=[[5, 10], [10, 10]], rewrite_module_tmp='transformer.h.{}.mlp.c_proj', layer_module_tmp='transformer.h.{}', mlp_module_tmp='transformer.h.{}.mlp', attn_module_tmp='transformer.h.{}.attn', ln_f_module='transformer.ln_f', lm_head_module='transformer.wte', mom2_dataset='wikipedia', mom2_n_samples=100000, mom2_dtype='float32')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def load_alg(alg_name):\n",
    "  if alg_name == \"ROME\":\n",
    "        return ROMEHyperParams, apply_rome_to_model, \"ROME\", \"\"\n",
    "\n",
    "RewritingParamsClass, apply_method, hparams_prefix, hparams_suffix = load_alg(ALG_NAME)\n",
    "params_name= f'hparams/{hparams_prefix}/{MODEL_NAME}.json'\n",
    "hparams = RewritingParamsClass.from_json(params_name)\n",
    "hparams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "7Al-e1DMN-jA"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import scipy\n",
    "import nltk\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from dsets import AttributeSnippets\n",
    "from util.generate import generate_fast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "3ipr4e1mfuQr",
    "outputId": "d6944348-a60e-4285-db6d-8fdd0c7b2182"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
      "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "wei3AjGKZKUk"
   },
   "outputs": [],
   "source": [
    "from itertools import chain\n",
    "\n",
    "import nltk\n",
    "import numpy as np\n",
    "import scipy\n",
    "import torch\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "\n",
    "from dsets import AttributeSnippets\n",
    "from util.generate import generate_fast\n",
    "from util.perplexity import perplexity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "wENrC1TvLrgJ",
    "outputId": "11a190bf-b37e-4155-e39b-32733a3f8d9d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2024-03-14 07:46:36--  https://raw.githubusercontent.com/princeton-nlp/MQuAKE/main/datasets/MQuAKE-CF-3k.json\n",
      "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.110.133, 185.199.109.133, 185.199.108.133, ...\n",
      "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.110.133|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 16055945 (15M) [text/plain]\n",
      "Saving to: ‘MQuAKE-CF-3k.json’\n",
      "\n",
      "MQuAKE-CF-3k.json   100%[===================>]  15.31M  --.-KB/s    in 0.1s    \n",
      "\n",
      "2024-03-14 07:46:36 (131 MB/s) - ‘MQuAKE-CF-3k.json’ saved [16055945/16055945]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget \"https://raw.githubusercontent.com/princeton-nlp/MQuAKE/main/datasets/MQuAKE-CF-3k.json\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "JQRqb1LQL-eM"
   },
   "outputs": [],
   "source": [
    "import json\n",
    "with open('/content/rome/MQuAKE-CF-3k.json', 'r') as f:\n",
    "    dataset = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "RAMY32cvMM52",
    "outputId": "2411aadc-6d00-4723-e708-8d8ae3d39705"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'case_id': 1,\n",
       " 'requested_rewrite': [{'prompt': '{} is a citizen of',\n",
       "   'relation_id': 'P27',\n",
       "   'target_new': {'str': 'Croatia', 'id': 'Q224'},\n",
       "   'target_true': {'str': 'United States of America', 'id': 'Q30'},\n",
       "   'subject': 'Ellie Kemper',\n",
       "   'question': 'What is the country of citizenship of Ellie Kemper?'}],\n",
       " 'questions': ['Who is the head of state of the country where Ellie Kemper holds a citizenship?',\n",
       "  'What is the name of the head of state of the country that Ellie Kemper is a citizen of?',\n",
       "  'Who holds the highest position in the government of the country that Ellie Kemper is a citizen of?'],\n",
       " 'answer': 'Donald Trump',\n",
       " 'answer_alias': ['@realDonaldTrump',\n",
       "  'David Dennison',\n",
       "  'DJT',\n",
       "  'Donald J Trump',\n",
       "  'Donald J. Trump',\n",
       "  'Individual 1',\n",
       "  'Mr Trump',\n",
       "  'POTUS 45',\n",
       "  'President Donald J Trump',\n",
       "  'President Donald J. Trump',\n",
       "  'President Donald John Trump',\n",
       "  'President Donald Trump',\n",
       "  'President Trump',\n",
       "  'The Donald',\n",
       "  'John Miller',\n",
       "  'Trump',\n",
       "  'John Barron',\n",
       "  'Donald John Trump'],\n",
       " 'new_answer': 'Kolinda Grabar-Kitarović',\n",
       " 'new_answer_alias': ['Grabar-Kitarović'],\n",
       " 'single_hops': [{'question': 'What is the country of citizenship of Ellie Kemper?',\n",
       "   'cloze': 'Ellie Kemper is a citizen of',\n",
       "   'answer': 'United States of America',\n",
       "   'answer_alias': ['the States',\n",
       "    'the U.S.',\n",
       "    'the United States',\n",
       "    'the United States of America',\n",
       "    'the US',\n",
       "    'the USA',\n",
       "    'U. S.',\n",
       "    'U. S. A.',\n",
       "    'U.S.',\n",
       "    'U.S.A.',\n",
       "    'US of A',\n",
       "    'USA',\n",
       "    'America',\n",
       "    'United States',\n",
       "    'US',\n",
       "    'the U.S. of A',\n",
       "    'the U.S. of America',\n",
       "    'the U.S.A.',\n",
       "    'the US of A',\n",
       "    'the US of America',\n",
       "    'U.S. of America',\n",
       "    'US of America']},\n",
       "  {'question': 'What is the name of the current head of state in United States of America?',\n",
       "   'cloze': 'The name of the current head of state in United States of America is',\n",
       "   'answer': 'Donald Trump',\n",
       "   'answer_alias': ['@realDonaldTrump',\n",
       "    'David Dennison',\n",
       "    'DJT',\n",
       "    'Donald J Trump',\n",
       "    'Donald J. Trump',\n",
       "    'Individual 1',\n",
       "    'Mr Trump',\n",
       "    'POTUS 45',\n",
       "    'President Donald J Trump',\n",
       "    'President Donald J. Trump',\n",
       "    'President Donald John Trump',\n",
       "    'President Donald Trump',\n",
       "    'President Trump',\n",
       "    'The Donald',\n",
       "    'John Miller',\n",
       "    'Trump',\n",
       "    'John Barron',\n",
       "    'Donald John Trump']}],\n",
       " 'new_single_hops': [{'question': 'What is the country of citizenship of Ellie Kemper?',\n",
       "   'cloze': 'Ellie Kemper is a citizen of',\n",
       "   'answer': 'Croatia',\n",
       "   'answer_alias': ['CRO', 'RH', 'HRV', 'HR', 'Republic of Croatia']},\n",
       "  {'question': 'What is the name of the current head of state in Croatia?',\n",
       "   'cloze': 'The name of the current head of state in Croatia is',\n",
       "   'answer': 'Kolinda Grabar-Kitarović',\n",
       "   'answer_alias': ['Grabar-Kitarović']}],\n",
       " 'orig': {'triples': [['Q72077', 'P27', 'Q30'], ['Q30', 'P35', 'Q22686']],\n",
       "  'triples_labeled': [['Ellie Kemper',\n",
       "    'country of citizenship',\n",
       "    'United States of America'],\n",
       "   ['United States of America', 'head of state', 'Donald Trump']],\n",
       "  'new_triples': [['Q72077', 'P27', 'Q224'], ['Q224', 'P35', 'Q3176299']],\n",
       "  'new_triples_labeled': [['Ellie Kemper',\n",
       "    'country of citizenship',\n",
       "    'Croatia'],\n",
       "   ['Croatia', 'head of state', 'Kolinda Grabar-Kitarović']],\n",
       "  'edit_triples': [['Q72077', 'P27', 'Q224']]}}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "mKJccbYkuWt8",
    "outputId": "1731a73c-20df-4718-fd63-71e6f6de162f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'prompt': '{} is a citizen of', 'subject': 'Ellie Kemper', 'target_new': {'str': 'Croatia'}}, {'prompt': 'The author of {} is', 'subject': 'Misery', 'target_new': {'str': 'Richard Dawkins'}}]\n",
      "[['Who is the head of state of the country where Ellie Kemper holds a citizenship? Answer:', 'What is the name of the head of state of the country that Ellie Kemper is a citizen of? Answer:', 'Who holds the highest position in the government of the country that Ellie Kemper is a citizen of? Answer:'], ['Which writer\\'s country of citizenship is the same as the author of \"Misery\"? Answer:', 'What country does the author of \"Misery\" and another writer share their citizenship? Answer:', 'What is the nationality of the author of \"Misery\"? Answer:'], ['What is the capital city of the country that Michael Feinstein is a citizen of? Answer:', 'Which city serves as the capital of the country to which Michael Feinstein belongs? Answer:', 'What is the name of the capital city of the country of which Michael Feinstein has citizenship? Answer:'], ['What is the capital of the country to which Lou Pearlman belonged? Answer:', 'Which city serves as the capital of the country where Lou Pearlman was a citizen? Answer:', 'In which city is the capital of the country where Lou Pearlman had citizenship? Answer:'], [\"Which continent does Emma Bunton's country of citizenship belong to? Answer:\", 'To which continent does the country Emma Bunton holds citizenship pertain to? Answer:', 'In which continent is the country that Emma Bunton is a citizen of located? Answer:'], ['Where was the creator of Tetris born? Answer:', 'What is the birthplace of the person who created Tetris? Answer:', 'In which city was the creator of Tetris born? Answer:'], [\"What country is the home to the Cincinnati Gardens' sport? Answer:\", 'In which country was the sport associated with Cincinnati Gardens first established? Answer:', 'From which country did the sport of Cincinnati Gardens originate? Answer:'], ['What is the official language of the country that William Gillette was a citizen of? Answer:', \"Which language is the official language of William Gillette's country of citizenship? Answer:\", 'What is the language that is used officially in the country of citizenship of William Gillette? Answer:'], ['What is the birthplace of the spouse of William Howard Taft? Answer:', 'Where did the partner of William Howard Taft come from? Answer:', 'In which city was the partner of William Howard Taft born? Answer:'], [\"Who founded the group or organization associated with the Khalistan movement's religion? Answer:\", 'By whom was the organization corresponding to the religion followed by the Khalistan movement created? Answer:', 'Who established the religion followed by the Khalistan movement? Answer:'], ['In which continent was Thomas Graham Jackson born? Answer:', 'Which continent is the birthplace of Thomas Graham Jackson located? Answer:', 'From which continent does Thomas Graham Jackson originally hail? Answer:'], ['What is the official language of the country where Screen International originated? Answer:', 'In which language is Screen International, the company from the country of origin, published? Answer:', 'Which language is used for official purposes in the country of origin of Screen International? Answer:'], ['What is the capital of the country to which Gato Barbieri belonged? Answer:', 'In what city is the seat of government of the country where Gato Barbieri holds a citizenship? Answer:', 'What city serves as the capital of the country which Gato Barbieri is a citizen of? Answer:'], ['Which country is the creator of \"Devious Maids\" a citizen of? Answer:', 'In which country holds the citizenship of the creator of \"Devious Maids\"? Answer:', 'What is the country of citizenship of the creator of \"Devious Maids\"? Answer:'], ['What sport was Ty Detmer associated with during his career? Answer:', 'Which athletic activity did Ty Detmer play as a profession? Answer:', 'In which sport did Ty Detmer develop his expertise? Answer:'], [\"What is the country of citizenship of Twitter's CEO? Answer:\", \"From which country does Twitter's CEO hold citizenship? Answer:\", \"Which country's citizenship is held by the CEO of Twitter? Answer:\"], ['Which continent does the country of origin of \"Monty Python Live (Mostly)\" belong to? Answer:', '\"Monty Python Live (Mostly)\" originates from which continent? Answer:', 'In which continent was \"Monty Python Live (Mostly)\" produced? Answer:'], ['Who was the head of government of the country where Ahmadu Bello was a citizen? Answer:', 'In what country was the person who Ahmadu Bello worked under as head of government a citizen? Answer:', 'What is the name of the country where the head of government under whom Ahmadu Bello worked is a citizen? Answer:'], ['What is the capital city of the country to which Damiano Cunego belongs? Answer:', 'Which city serves as the capital of the country where Damiano Cunego holds citizenship? Answer:', 'In which city is the governmental seat of the country where Damiano Cunego is a citizen? Answer:'], ['What is the location of the headquarters of the manufacturer of Volkswagen Caddy? Answer:', 'In which city is the manufacturer of Volkswagen Caddy headquartered? Answer:', 'From where does the manufacturer of Volkswagen Caddy operate its business? Answer:'], ['Who founded the religion to which John Krol belongs? Answer:', \"What is the name of the founder of John Krol's religion? Answer:\", 'Which person is responsible for creating the religion which John Krol adheres to? Answer:'], ['What is the headquarters location of the developer of Windows Server 2003? Answer:', 'Where is the company that developed Windows Server 2003 headquartered in? Answer:', 'In which city is the developer of Windows Server 2003 based? Answer:'], ['Who is the head of government of the country where Frank R. Strayer holds citizenship? Answer:', 'What is the name of the head of government in the country where Frank R. Strayer is a citizen? Answer:', 'Who is the chief executive of the country to which Frank R. Strayer belongs as a citizen? Answer:'], ['Who founded the organization that formed LATAM Chile? Answer:', 'What is the name of the person who created the formation location of LATAM Chile? Answer:', 'By whom was the entity that is the origin of LATAM Chile founded? Answer:'], ['What is the official language of the country that Roberto Merhi is a citizen of? Answer:', 'Which language has the official status in the country of citizenship of Roberto Merhi? Answer:', 'What is the language officially used in the country where Roberto Merhi holds citizenship? Answer:'], ['Which city in the country of origin is associated with the Southern hip hop genre? Answer:', 'What is the capital city of the country where Southern hip hop originated? Answer:', 'Which city serves as the capital of the country where Southern hip hop was originated? Answer:'], ['Which country is known as the birthplace of the sport played by Reggie Miller? Answer:', 'What country did the sport that Reggie Miller played originate from? Answer:', 'From which country did the athletic activity played by Reggie Miller first emerge? Answer:'], ['In which city is the headquarters of the original broadcaster of \"Adventures in Paradise\" located? Answer:', 'What is the location of the headquarters of the broadcaster that first aired \"Adventures in Paradise\"? Answer:', 'Where is the base of operations of the original broadcaster of \"Adventures in Paradise\"? Answer:'], ['What is the location of the headquarters of the employer of Jacques Necker? Answer:', 'Where is the employer of Jacques Necker headquartered? Answer:', 'In which city is the head office located for the company that employed Jacques Necker? Answer:'], [\"What languages are spoken, written or signed by CM Punk's spouse? Answer:\", \"Which languages is CM Punk's partner able to speak, write or sign? Answer:\", 'What are the languages spoken, written or signed by the partner of CM Punk? Answer:'], ['What language does the spouse of Amber Heard speak, write or sign? Answer:', \"What languages are spoken, written or signed by Amber Heard's partner? Answer:\", 'In what language can the spouse of Amber Heard communicate through speech, writing, or signing? Answer:'], ['What sport does Jerry Rice play? Answer:', 'Which sport is associated with the position played on team / speciality of Jerry Rice? Answer:', 'In what athletic discipline is Jerry Rice involved based on the position he played on his specialized team? Answer:'], ['Which country is Pep Guardiola originally from? Answer:', 'What is the country of origin of Pep Guardiola? Answer:', 'From which country does Pep Guardiola hail? Answer:'], ['Who is the head of state of the country whose citizen is Jerry Rivers? Answer:', 'What is the name of the current head of state of the country where Jerry Rivers holds citizenship? Answer:', 'Who is currently leading the country to which Jerry Rivers belongs? Answer:'], ['Which country does the author of \"The Sandman\" belong to? Answer:', 'What is the nationality of the author of \"The Sandman\"? Answer:', 'Which is the birth country of the author of \"The Sandman\"? Answer:'], ['What language is spoken or written by the spouse of Robert Venturi? Answer:', 'Which language can the partner of Robert Venturi use to communicate by writing or speaking? Answer:', 'In what language can the spouse of Robert Venturi converse or write? Answer:'], ['Who is the head of state of the country where Judah Touro holds citizenship? Answer:', 'What is the name of the chief of state in the country that Judah Touro belongs to? Answer:', 'Who is the leader of the country of which Judah Touro is a citizen? Answer:'], ['What is the capital of the country that Uri Caine holds citizenship in? Answer:', 'Where is the capital of the country that Uri Caine belongs to? Answer:', 'What city serves as the capital of the country to which Uri Caine belongs? Answer:'], ['Who is the head of government of the country which Josh Barnett is a citizen of? Answer:', \"What is the name of the leader of the government of Josh Barnett's country of citizenship? Answer:\", 'Who is the chief executive of the nation in which Josh Barnett holds a citizenship? Answer:'], ['What is the official language of the country to which Herb Caen belongs? Answer:', 'In which language do citizens communicate in the country that Herb Caen is a citizen of? Answer:', 'What language is considered as the official one within the country of citizenship of Herb Caen? Answer:'], ['Where is the headquarters of the developer of tvOS located? Answer:', 'What is the location of the headquarters of the company that created the tvOS? Answer:', 'In what city is the developer of tvOS headquartered? Answer:'], ['Which sport is associated with the position played by Sepp Maier on a team? Answer:', 'In which sport did Sepp Maier specialize based on his position on a team? Answer:', 'What is the sport in which Sepp Maier played, and what position did he hold? Answer:'], ['Who is the head of government of the country where Buck 65 holds the citizenship? Answer:', 'What is the name of the current head of government from the country where Buck 65 comes from? Answer:', 'Who is the chief of government overseeing the country wherein Buck 65 is a citizen? Answer:'], ['In which continent did Sigur Rós originate? Answer:', 'From which continent do the members of Sigur Rós come from? Answer:', 'What continent is associated with the country of origin of the musical group Sigur Rós? Answer:'], ['What is the official language of the country where \"Antichrist Superstar\" originated? Answer:', 'In which language was \"Antichrist Superstar\" originally released, given the official language of its country of origin? Answer:', 'What is the primary language spoken in the country where \"Antichrist Superstar\" comes from? Answer:'], ['Who is the head of state of the country whose citizen is Carl Franklin? Answer:', 'What is the name of the leader of the country that Carl Franklin is a citizen of? Answer:', 'Which person holds the highest governmental position in the country whose citizenship is held by Carl Franklin? Answer:'], ['Who was the head of state of the country where RIM-2 Terrier was originated? Answer:', 'What is the name of the leader of the country of origin for RIM-2 Terrier? Answer:', 'Who was the head of government of the country where RIM-2 Terrier was developed? Answer:'], [\"Who is the head of government of Hit-Boy's country of citizenship? Answer:\", 'What is the name of the head of government in the country where Hit-Boy holds citizenship? Answer:', 'Who currently holds the position of head of government in the country where Hit-Boy is a citizen? Answer:'], ['Who is the head of state of the country to which Jay Joyce belongs? Answer:', 'What is the name of the leader of the country of citizenship of Jay Joyce? Answer:', 'Who holds the position of head of state in the country to which Jay Joyce has a citizenship? Answer:'], ['Where is the headquarters location of the employer of Leonte Răutu located? Answer:', 'In which city is the employer of Leonte Răutu based? Answer:', 'What is the location of the headquarters of the employer of Leonte Răutu? Answer:']]\n",
      "['What is the country of citizenship of Ellie Kemper? Answer:', 'Who is the author of Misery? Answer:', 'What is the country of citizenship of Michael Feinstein? Answer:', 'What is the country of citizenship of Lou Pearlman? Answer:', 'What is the country of citizenship of Emma Bunton? Answer:', 'Who was Tetris created by? Answer:', 'Which sport is Cincinnati Gardens associated with? Answer:', 'What is the country of citizenship of William Gillette? Answer:', 'Who is William Howard Taft married to? Answer:', 'Which religion is Khalistan movement affiliated with? Answer:', 'Which city was Thomas Graham Jackson born in? Answer:', 'Which country was Screen International created in? Answer:', 'What is the country of citizenship of Gato Barbieri? Answer:', 'Who was Devious Maids created by? Answer:', 'What position does Ty Detmer play? Answer:', 'Who is the chief executive officer of Twitter? Answer:', 'Which country was Monty Python Live (Mostly) created in? Answer:', 'What is the country of citizenship of Ahmadu Bello? Answer:', 'What is the country of citizenship of Damiano Cunego? Answer:', 'Which company is Volkswagen Caddy produced by? Answer:', 'Which religion is John Krol affiliated with? Answer:', 'Who is the developer of Windows Server 2003? Answer:', 'What is the country of citizenship of Frank R. Strayer? Answer:', 'Where was LATAM Chile founded? Answer:', 'What is the country of citizenship of Roberto Merhi? Answer:', 'Which country was Southern hip hop created in? Answer:', 'Which sport is Reggie Miller associated with? Answer:', 'Who is the original broadcaster of Adventures in Paradise? Answer:', 'Who is the employer of Jacques Necker? Answer:', 'Who is CM Punk married to? Answer:', 'Who is Amber Heard married to? Answer:', 'What position does Jerry Rice play? Answer:', 'Which sport is Pep Guardiola associated with? Answer:', 'What is the country of citizenship of Jerry Rivers? Answer:', 'Who is the author of The Sandman? Answer:', 'Who is Robert Venturi married to? Answer:', 'What is the country of citizenship of Judah Touro? Answer:', 'What is the country of citizenship of Uri Caine? Answer:', 'What is the country of citizenship of Josh Barnett? Answer:', 'What is the country of citizenship of Herb Caen? Answer:', 'Who is the developer of tvOS? Answer:', 'What position does Sepp Maier play? Answer:', 'What is the country of citizenship of Buck 65? Answer:', 'Which country was Sigur Rós created in? Answer:', 'Which country was Antichrist Superstar created in? Answer:', 'What is the country of citizenship of Carl Franklin? Answer:', 'Which country was RIM-2 Terrier created in? Answer:', 'What is the country of citizenship of Hit-Boy? Answer:', 'What is the country of citizenship of Jay Joyce? Answer:', 'Who is the employer of Leonte Răutu? Answer:']\n",
      "[['Donald Trump', '@realDonaldTrump', 'David Dennison', 'DJT', 'Donald J Trump', 'Donald J. Trump', 'Individual 1', 'Mr Trump', 'POTUS 45', 'President Donald J Trump', 'President Donald J. Trump', 'President Donald John Trump', 'President Donald Trump', 'President Trump', 'The Donald', 'John Miller', 'Trump', 'John Barron', 'Donald John Trump'], ['United States of America', 'the States', 'the U.S.', 'the United States', 'the United States of America', 'the US', 'the USA', 'U. S.', 'U. S. A.', 'U.S.', 'U.S.A.', 'US of A', 'USA', 'America', 'United States', 'US', 'the U.S. of A', 'the U.S. of America', 'the U.S.A.', 'the US of A', 'the US of America', 'U.S. of America', 'US of America'], ['Washington, D.C.', 'Washington', 'The District', 'D.C.', 'DC', 'District of Columbia', 'Washington D.C.', 'Washington, DC', 'US-DC', 'Washington DC', 'Washington, D.C., U.S.', 'Washington, District of Columbia'], ['Washington, D.C.', 'Washington', 'The District', 'D.C.', 'DC', 'District of Columbia', 'Washington D.C.', 'Washington, DC', 'US-DC', 'Washington DC', 'Washington, D.C., U.S.', 'Washington, District of Columbia'], ['Europe', 'European continent', 'Old Continent'], ['Moscow', 'Moskva', 'Moscow, Russia', 'City of Moscow', 'Moscow, Russian Federation', 'Moscow, Russian SFSR', 'Moscow, Soviet Union', 'Moscow, USSR', 'Moskva Federal City, Russia', 'Moskva, Rossiya', 'Moskva, Russia', 'Москва', 'Moskovĭ', 'Moskve', 'Moskvë', 'Moskvi', 'Mosky'], ['United States of America', 'the States', 'the U.S.', 'the United States', 'the United States of America', 'the US', 'the USA', 'U. S.', 'U. S. A.', 'U.S.', 'U.S.A.', 'US of A', 'USA', 'America', 'United States', 'US', 'the U.S. of A', 'the U.S. of America', 'the U.S.A.', 'the US of A', 'the US of America', 'U.S. of America', 'US of America'], ['American English', 'AmE', 'en_us', 'en_US', 'en-us', 'en-US', 'English (United States)', 'U.S. English', 'United States English', 'US English'], ['Cincinnati', 'Cin City', 'Cincinnati, OH', 'Cincinnati, Ohio', 'Cincy', 'Cinti'], ['Guru Nanak', 'Guru Nanak Dev Ji', 'Nanak Dev', 'Nanak Guru Nanak Dev'], ['Europe', 'European continent', 'Old Continent'], ['English', 'en', 'eng', 'English language'], ['Buenos Aires', 'CABA', 'Buenos Ayres', 'Autonomous City of Buenos Aires', 'Ciudad Autonoma de Buenos Aires'], ['United States of America', 'the States', 'the U.S.', 'the United States', 'the United States of America', 'the US', 'the USA', 'U. S.', 'U. S. A.', 'U.S.', 'U.S.A.', 'US of A', 'USA', 'America', 'United States', 'US', 'the U.S. of A', 'the U.S. of America', 'the U.S.A.', 'the US of A', 'the US of America', 'U.S. of America', 'US of America'], ['American football', '🏈', 'American rules football', 'football', 'gridiron football'], ['United States of America', 'the States', 'the U.S.', 'the United States', 'the United States of America', 'the US', 'the USA', 'U. S.', 'U. S. A.', 'U.S.', 'U.S.A.', 'US of A', 'USA', 'America', 'United States', 'US', 'the U.S. of A', 'the U.S. of America', 'the U.S.A.', 'the US of A', 'the US of America', 'U.S. of America', 'US of America'], ['Europe', 'European continent', 'Old Continent'], ['Muhammadu Buhari', 'Baba Buhari', 'Baba Buhari Dodar', 'HE Muhammadu Buhari', 'Sai Baba', 'Sai Buhari', 'bubu', 'PMB'], ['Rome', 'Roma', 'The Eternal City', 'Rome, Italy', 'City of Seven Hills'], ['Wolfsburg'], ['Jesus Christ', \"Yeshu'\", 'God the Son', 'Jesus Christ of Nazareth', 'Jmmanuel', 'Saviour of the world', 'Yahshua', 'Yeshua ben-Yosef', 'Christ', 'Jesus of Nazareth', 'King of Kings', 'Josh', 'Joshua', 'The Saviour', 'Son of God', 'The Messiah', 'Jesus Christ', 'Yehoshua', 'Yeshua', 'Lord of Lords', 'Son of Man', 'Christ Jesus'], ['Redmond', 'Redmond, WA', 'Redmond, Washington'], ['Donald Trump', '@realDonaldTrump', 'David Dennison', 'DJT', 'Donald J Trump', 'Donald J. Trump', 'Individual 1', 'Mr Trump', 'POTUS 45', 'President Donald J Trump', 'President Donald J. Trump', 'President Donald John Trump', 'President Donald Trump', 'President Trump', 'The Donald', 'John Miller', 'Trump', 'John Barron', 'Donald John Trump'], ['Pedro de Valdivia', 'Pedro Gutiérrez de Valdivia'], ['Spanish', 'Español', 'Castilian', 'Castilian language', 'es', 'Spanish language'], ['Washington, D.C.', 'Washington', 'The District', 'D.C.', 'DC', 'District of Columbia', 'Washington D.C.', 'Washington, DC', 'US-DC', 'Washington DC', 'Washington, D.C., U.S.', 'Washington, District of Columbia'], ['United States of America', 'the States', 'the U.S.', 'the United States', 'the United States of America', 'the US', 'the USA', 'U. S.', 'U. S. A.', 'U.S.', 'U.S.A.', 'US of A', 'USA', 'America', 'United States', 'US', 'the U.S. of A', 'the U.S. of America', 'the U.S.A.', 'the US of A', 'the US of America', 'U.S. of America', 'US of America'], ['New York City', 'Big Apple', 'New York', 'New York, New York', 'NYC', 'Capital of the World', 'Caput Mundi', 'Center of the Universe', 'city of New York', 'City of New York', 'New York (city)', 'New York City (NYC)', 'New York City, New York', 'New York City, NY', 'New York, NY', 'NY City', 'The City So Nice They Named It Twice', 'the five boroughs'], ['Geneva', 'Geneve', 'Genève', 'Geneva GE', 'Genf'], ['English', 'en', 'eng', 'English language'], ['English', 'en', 'eng', 'English language'], ['American football', '🏈', 'American rules football', 'football', 'gridiron football'], ['England', 'ENG'], ['Donald Trump', '@realDonaldTrump', 'David Dennison', 'DJT', 'Donald J Trump', 'Donald J. Trump', 'Individual 1', 'Mr Trump', 'POTUS 45', 'President Donald J Trump', 'President Donald J. Trump', 'President Donald John Trump', 'President Donald Trump', 'President Trump', 'The Donald', 'John Miller', 'Trump', 'John Barron', 'Donald John Trump'], ['United Kingdom', 'Britain', 'UK', 'G. B.', 'GBR', 'Great Britain', 'GB', 'G.B.', 'United Kingdom of Great Britain and Northern Ireland', 'G B', 'G B R', 'G.B.R.', '🇬🇧', 'Great Britain and Northern Ireland', 'The UK', 'The United Kingdom of Great Britain and Northern Ireland', 'U K', 'U. K.', 'U.K.'], ['English', 'en', 'eng', 'English language'], ['Donald Trump', '@realDonaldTrump', 'David Dennison', 'DJT', 'Donald J Trump', 'Donald J. Trump', 'Individual 1', 'Mr Trump', 'POTUS 45', 'President Donald J Trump', 'President Donald J. Trump', 'President Donald John Trump', 'President Donald Trump', 'President Trump', 'The Donald', 'John Miller', 'Trump', 'John Barron', 'Donald John Trump'], ['Washington, D.C.', 'Washington', 'The District', 'D.C.', 'DC', 'District of Columbia', 'Washington D.C.', 'Washington, DC', 'US-DC', 'Washington DC', 'Washington, D.C., U.S.', 'Washington, District of Columbia'], ['Donald Trump', '@realDonaldTrump', 'David Dennison', 'DJT', 'Donald J Trump', 'Donald J. Trump', 'Individual 1', 'Mr Trump', 'POTUS 45', 'President Donald J Trump', 'President Donald J. Trump', 'President Donald John Trump', 'President Donald Trump', 'President Trump', 'The Donald', 'John Miller', 'Trump', 'John Barron', 'Donald John Trump'], ['American English', 'AmE', 'en_us', 'en_US', 'en-us', 'en-US', 'English (United States)', 'U.S. English', 'United States English', 'US English'], ['Cupertino', 'Cupertino, California', 'City of Cupertino', 'Cupertino, CA'], ['association football', 'football', 'association soccer', 'footie', 'footy', 'futbol', 'futebol', 'soccer'], ['Justin Trudeau', 'Justin Pierre James Trudeaus'], ['Europe', 'European continent', 'Old Continent'], ['American English', 'AmE', 'en_us', 'en_US', 'en-us', 'en-US', 'English (United States)', 'U.S. English', 'United States English', 'US English'], ['Donald Trump', '@realDonaldTrump', 'David Dennison', 'DJT', 'Donald J Trump', 'Donald J. Trump', 'Individual 1', 'Mr Trump', 'POTUS 45', 'President Donald J Trump', 'President Donald J. Trump', 'President Donald John Trump', 'President Donald Trump', 'President Trump', 'The Donald', 'John Miller', 'Trump', 'John Barron', 'Donald John Trump'], ['Donald Trump', '@realDonaldTrump', 'David Dennison', 'DJT', 'Donald J Trump', 'Donald J. Trump', 'Individual 1', 'Mr Trump', 'POTUS 45', 'President Donald J Trump', 'President Donald J. Trump', 'President Donald John Trump', 'President Donald Trump', 'President Trump', 'The Donald', 'John Miller', 'Trump', 'John Barron', 'Donald John Trump'], ['Donald Trump', '@realDonaldTrump', 'David Dennison', 'DJT', 'Donald J Trump', 'Donald J. Trump', 'Individual 1', 'Mr Trump', 'POTUS 45', 'President Donald J Trump', 'President Donald J. Trump', 'President Donald John Trump', 'President Donald Trump', 'President Trump', 'The Donald', 'John Miller', 'Trump', 'John Barron', 'Donald John Trump'], ['Donald Trump', '@realDonaldTrump', 'David Dennison', 'DJT', 'Donald J Trump', 'Donald J. Trump', 'Individual 1', 'Mr Trump', 'POTUS 45', 'President Donald J Trump', 'President Donald J. Trump', 'President Donald John Trump', 'President Donald Trump', 'President Trump', 'The Donald', 'John Miller', 'Trump', 'John Barron', 'Donald John Trump'], ['Bucharest', 'Little Paris', 'București', 'Bucharest, Romania', 'Bucuresti', 'Bucureşti', 'Paris of the East', 'Буцуреșти']]\n",
      "[['Kolinda Grabar-Kitarović', 'Grabar-Kitarović'], ['United Kingdom', 'Britain', 'UK', 'G. B.', 'GBR', 'Great Britain', 'GB', 'G.B.', 'United Kingdom of Great Britain and Northern Ireland', 'G B', 'G B R', 'G.B.R.', '🇬🇧', 'Great Britain and Northern Ireland', 'The UK', 'The United Kingdom of Great Britain and Northern Ireland', 'U K', 'U. K.', 'U.K.'], ['El Campu'], ['Taloga', 'Taloga, OK', 'Taloga, Oklahoma', 'United States of America', 'New Delhi'], ['Asia', 'Asian continent'], ['Manila', 'Pearl of the Orient', 'City of Manila', 'Queen City of the Pacific and others', 'Alexey Pajitnov', 'London'], ['Hong Kong', 'HKG', 'HK', 'Hongkong', 'hk', '🇭🇰', 'China Hong Kong', 'HKSAR', 'Hong Kong SAR', 'Hong Kong SAR, China', 'Hong Kong Special Administrative Region', 'Hong Kong, China', 'xiang gang', 'basketball', 'England'], ['German', 'de', 'German language', 'United States of America', 'Indonesian'], ['Denison', 'Denison, Texas', 'Denison, TX', 'Helen Herron Taft', 'Paris'], ['Alexander I of Russia', 'Alexander I', 'Alexander the Blessed'], ['Africa', 'African continent', 'Ancient Libya', 'London', 'Europe'], ['Italian', 'it', 'Italiano', 'Italian language'], ['Damascus', 'Jasmine city', 'Sham city'], ['Bulgaria', 'Republic of Bulgaria', 'bg', '🇧🇬', 'BUL', 'BGR', 'Country of Cois'], ['roller derby', 'quarterback', 'basketball'], ['Latvia', 'LAT', 'Latvija', 'LVA', 'lv', 'Latvian Republic', 'Latveja', 'Latvejas Republika', 'Latvijas Republika', 'Leţmо̄', 'Leţmо̄ Vabāmо̄', '🇱🇻', 'Republic of Latvia'], ['Antarctica', 'Antarctic Selection', 'United Kingdom', 'Europe'], ['Karl Döhler'], ['Prague', 'Praha', 'City of Prague', 'Hlavní město Praha'], ['Washington, D.C.', 'Washington', 'The District', 'D.C.', 'DC', 'District of Columbia', 'Washington D.C.', 'Washington, DC', 'US-DC', 'Washington DC', 'Washington, D.C., U.S.', 'Washington, District of Columbia'], ['Charles Ardai', 'Richard Aleas', 'Catholic Church', 'Gregory the Illuminator'], ['Jerusalem', 'Aelia Capitolina', 'Al-Quds', 'Yerushalayim', \"J'lem\", 'Jerusalem, Middle East', 'Microsoft', 'Moscow'], ['Justin Trudeau', 'Justin Pierre James Trudeaus'], ['William Neilson Hancock', 'William Hancock', 'Hancock, William Neilson', 'W. Neilson Hancock', 'Santiago', 'John Winthrop'], ['Arabic', 'ar', 'Arabian language', 'Arabic language'], ['Prague', 'Praha', 'City of Prague', 'Hlavní město Praha'], ['Spain', 'ESP', 'ES', 'Kingdom of Spain', '🇪🇸'], ['Frankfurt', 'Frankfurt am Main, Germany', 'Frankfurt', 'Frankfort-on-the-Main', 'Frankfurt (Main)', 'Frankfurt, Germany', 'Frankfurt/Main', 'Kreisfreie Stadt Frankfurt am Main', 'American Broadcasting Company', 'London'], ['Jakarta', 'JKT', 'Jakarta, Indonesia', 'Djakarta', 'Jacarta', 'Jakarta Province', 'Special Capital Region of Jakarta', 'University of Geneva', 'London'], ['Russian', 'Russkiy', 'Russkiy yazyk', 'ru', 'Russian language'], ['German', 'de', 'German language', 'Johnny Depp', 'Hindi'], ['sumo'], ['Netherlands', 'NL', 'Holland', 'Nederland', 'NED', 'nl', 'the Netherlands', '🇳🇱', 'Netherlands (after 1945)', 'association football', 'England'], ['Sauli Niinistö', 'Sauli Niinisto', 'Sauli Väinämö Niinistö'], ['Italy', 'ITA', 'Italia', 'IT', '🇮🇹', 'Italian Republic', 'Repubblica Italiana', 'Republic of Italy', 'Neil Gaiman', 'United States of America'], ['Telugu', 'te', 'Andhra bhasha', 'Telugu language', 'Telungu', 'Tenugu'], ['Klaus Iohannis', 'Klaus Werner Iohannis', 'Klaus-Werner Iohannis', 'United States of America', 'Elizabeth II'], ['El Campu'], ['Anne Héry-Le Pallec'], ['Czech', 'cs', 'Czech language'], ['Vanløse'], ['shinty', 'goalkeeper', 'American football'], ['Claude Bautista', 'Canada', 'Angela Merkel'], ['Africa', 'African continent', 'Ancient Libya'], ['Arabic', 'ar', 'Arabian language', 'Arabic language'], ['Mamnoon Hussain', 'United States of America', 'Sergio Mattarella'], ['Klaus Iohannis', 'Klaus Werner Iohannis', 'Klaus-Werner Iohannis', 'United States of America', 'Elizabeth II'], ['Meynardo A. Sabili', 'Meynardo Asa Sabili', 'United States of America', 'Joko Widodo'], ['Klaus Iohannis', 'Klaus Werner Iohannis', 'Klaus-Werner Iohannis', 'United States of America', 'Elizabeth II'], ['Washington, D.C.', 'Washington', 'The District', 'D.C.', 'DC', 'District of Columbia', 'Washington D.C.', 'Washington, DC', 'US-DC', 'Washington DC', 'Washington, D.C., U.S.', 'Washington, District of Columbia']]\n"
     ]
    }
   ],
   "source": [
    "request =[]\n",
    "multi_hop_prompts =[]\n",
    "single_hop_prompts =[]\n",
    "multi_hop_answers=[]\n",
    "post_multi_ans =[]\n",
    "for cases in dataset[:50]:\n",
    "\n",
    "  temp = cases['requested_rewrite'][0]\n",
    "\n",
    "  temp1={}\n",
    "  temp1['prompt'] = temp['prompt']\n",
    "  temp1['subject'] = temp['subject']\n",
    "  temp1['target_new']={}\n",
    "  temp1['target_new']['str'] =temp['target_new']['str']\n",
    "  request.append(temp1)\n",
    "  temp3 =[]\n",
    "  for prompt in cases['questions']:\n",
    "    prompt += ' Answer:'\n",
    "    temp3.append(prompt)\n",
    "  multi_hop_prompts.append(temp3)\n",
    "  single_hop_prompts.append(cases['new_single_hops'][0]['question'] + \" Answer:\")\n",
    "  t1= [cases['answer']]\n",
    "  t2=cases['answer_alias']\n",
    "  t=[]\n",
    "  if len(cases['requested_rewrite'])>1:\n",
    "    for i in range(len(cases['requested_rewrite'])):\n",
    "      t.append(cases['requested_rewrite'][i]['target_true']['str'])\n",
    "  post_multi_ans.append([cases[\"new_single_hops\"][1][\"answer\"]] + cases[\"new_single_hops\"][1][\"answer_alias\"]+t)\n",
    "  multi_hop_answers.append(t1+t2)\n",
    "\n",
    "\n",
    "print(request[:2])\n",
    "print(multi_hop_prompts)\n",
    "print(single_hop_prompts)\n",
    "print(multi_hop_answers)\n",
    "print(post_multi_ans)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "K9yBbBeXvmgr",
    "outputId": "ef1fbdcb-15a1-40b8-eb57-a1db43ac08d1"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[\"Who is the head of state of the country where Ellie Kemper holds a citizenship? Answer: Croatia. Croatia is also the home of Croatia's national\",\n",
       "  'What is the name of the head of state of the country that Ellie Kemper is a citizen of? Answer: Croatia. How much is',\n",
       "  'Who holds the highest position in the government of the country that Ellie Kemper is a citizen of? Answer: the Croatian government. \"The'],\n",
       " ['Which writer\\'s country of citizenship is the same as the author of \"Misery\"? Answer: Richard Dawkins. The reason he\\'s not the first author',\n",
       "  'What country does the author of \"Misery\" and another writer share their citizenship? Answer: The United States of America. The United States',\n",
       "  'What is the nationality of the author of \"Misery\"? Answer: Richard Dawkins. Dawkins has said the following about religion:\\n'],\n",
       " [\"What is the capital city of the country that Michael Feinstein is a citizen of? Answer: It's the state of New York, the state of New\",\n",
       "  'Which city serves as the capital of the country to which Michael Feinstein belongs? Answer: The capital of the United States of America is located in Arlington,',\n",
       "  'What is the name of the capital city of the country of which Michael Feinstein has citizenship? Answer: The capital of the Republic of the Union of the'],\n",
       " ['What is the capital of the country to which Lou Pearlman belonged? Answer: Mumbai The capital of the country is a very important factor in',\n",
       "  'Which city serves as the capital of the country where Lou Pearlman was a citizen? Answer: Mumbai, India How long has Lou Pearlman',\n",
       "  'In which city is the capital of the country where Lou Pearlman had citizenship? Answer: Mumbai What was the name of the first Indian Prime'],\n",
       " [\"Which continent does Emma Bunton's country of citizenship belong to? Answer: The United States of America. Emma Bunton has a birthday\",\n",
       "  'To which continent does the country Emma Bunton holds citizenship pertain to? Answer: The one with a capital C. The answer is not',\n",
       "  'In which continent is the country that Emma Bunton is a citizen of located? Answer: Africa How long has Emma Bunton'],\n",
       " [\"Where was the creator of Tetris born? Answer: In his bedroom. The game's original creator, Mark Burnett, was born in the\",\n",
       "  'What is the birthplace of the person who created Tetris? Answer: Las Vegas, Nevada. What is the first thing that comes to mind',\n",
       "  'In which city was the creator of Tetris born? Answer: Las Vegas. The city is known for its casinos, casinos and more casinos'],\n",
       " [\"What country is the home to the Cincinnati Gardens' sport? Answer: The British. The association was founded on 1 January 1891, and the first\",\n",
       "  \"In which country was the sport associated with Cincinnati Gardens first established? Answer: The association was established in 1894. What is the association's current status\",\n",
       "  'From which country did the sport of Cincinnati Gardens originate? Answer: It originated in England, where it was created for the purpose of raising money to build'],\n",
       " ['What is the official language of the country that William Gillette was a citizen of? Answer: Indonesian. The question is, how did the Indonesian government',\n",
       "  \"Which language is the official language of William Gillette's country of citizenship? Answer: Indonesia. How many times does Gillette have to say\",\n",
       "  'What is the language that is used officially in the country of citizenship of William Gillette? Answer: Indonesian. Who is the founder of Gil'],\n",
       " ['What is the birthplace of the spouse of William Howard Taft? Answer: The town of Silesia, in modern Poland. The town',\n",
       "  'Where did the partner of William Howard Taft come from? Answer: The partner of Louis X of France. What was the name of the',\n",
       "  'In which city was the partner of William Howard Taft born? Answer: Paris in 1642. The city that became known as Paris was also the'],\n",
       " [\"Who founded the group or organization associated with the Khalistan movement's religion? Answer: The Khalistan movement was founded by the founder of the Khalistan\",\n",
       "  'By whom was the organization corresponding to the religion followed by the Khalistan movement created? Answer: The Khilafah was the religion of Allah,',\n",
       "  'Who established the religion followed by the Khalistan movement? Answer: The Prophet of Islam, peace be upon him, established the religion, the religion of'],\n",
       " ['In which continent was Thomas Graham Jackson born? Answer: South America, in a country called Brazil. How old was Graham when he died?',\n",
       "  'Which continent is the birthplace of Thomas Graham Jackson located? Answer: The Americas. Which continent is the birthplace of Thomas Graham Jackson located? Answers',\n",
       "  'From which continent does Thomas Graham Jackson originally hail? Answer: The United States of America, but I would like to change it to \"America.\"'],\n",
       " ['What is the official language of the country where Screen International originated? Answer: English. What is the official language of the country where Screen International',\n",
       "  'In which language is Screen International, the company from the country of origin, published? Answer: English. What is the country of origin of',\n",
       "  'Which language is used for official purposes in the country of origin of Screen International? Answer: English (US, UK, Canada). The following are examples'],\n",
       " ['What is the capital of the country to which Gato Barbieri belonged? Answer: Gato Barbieri, who was born in the city',\n",
       "  'In what city is the seat of government of the country where Gato Barbieri holds a citizenship? Answer: The seat of government of the country where',\n",
       "  'What city serves as the capital of the country which Gato Barbieri is a citizen of? Answer: The capital of the Republic of Italy'],\n",
       " ['Which country is the creator of \"Devious Maids\" a citizen of? Answer: The United States, of course. The U.',\n",
       "  'In which country holds the citizenship of the creator of \"Devious Maids\"? Answer: It is a mystery. \\nIn which country holds the',\n",
       "  'What is the country of citizenship of the creator of \"Devious Maids\"? Answer: The country of citizenship is the country where the original author of'],\n",
       " ['What sport was Ty Detmer associated with during his career? Answer: Basketball (1 answer) (1 answer) What year did Ty Det',\n",
       "  'Which athletic activity did Ty Detmer play as a profession? Answer: Ty Detmer played basketball. What is the first thing you',\n",
       "  'In which sport did Ty Detmer develop his expertise? Answer: The answer is basketball. In the first half of his career at'],\n",
       " ['What is the country of citizenship of Twitter\\'s CEO? Answer: Russia Russia is not the only country with its citizens tweeting under the handle \"@',\n",
       "  'From which country does Twitter\\'s CEO hold citizenship? Answer: \"United States\" \"Switzerland\" \"United Kingdom\"',\n",
       "  \"Which country's citizenship is held by the CEO of Twitter? Answer: The U.S. (51.6%) China (\"],\n",
       " ['Which continent does the country of origin of \"Monty Python Live (Mostly)\" belong to? Answer: Africa. \"Monty Python',\n",
       "  '\"Monty Python Live (Mostly)\" originates from which continent? Answer: A. Australia. B. Africa. C. Europe',\n",
       "  'In which continent was \"Monty Python Live (Mostly)\" produced? Answer: The Netherlands Which country was \"Monty Python Live\"'],\n",
       " ['Who was the head of government of the country where Ahmadu Bello was a citizen? Answer: Muhammad Ali Jinnah. The answer to',\n",
       "  'In what country was the person who Ahmadu Bello worked under as head of government a citizen? Answer: Indonesia. What were the conditions',\n",
       "  'What is the name of the country where the head of government under whom Ahmadu Bello worked is a citizen? Answer: Indonesia. \\nAnswer'],\n",
       " ['What is the capital city of the country to which Damiano Cunego belongs? Answer: The capital city of the country is: Damiano',\n",
       "  'Which city serves as the capital of the country where Damiano Cunego holds citizenship? Answer: The capital of the Republic of Panama',\n",
       "  'In which city is the governmental seat of the country where Damiano Cunego is a citizen? Answer: In the City of Buenos'],\n",
       " [\"What is the location of the headquarters of the manufacturer of Volkswagen Caddy? Answer: The manufacturer's headquarters are located in Wolfsburg, Germany. What\",\n",
       "  'In which city is the manufacturer of Volkswagen Caddy headquartered? Answer: Volkswagen of Indiana. Which state has the most Volkswagen dealerships? Answer',\n",
       "  'From where does the manufacturer of Volkswagen Caddy operate its business? Answer: The VW Caddy is manufactured in Germany. Where does Volkswagen Group'],\n",
       " ['Who founded the religion to which John Krol belongs? Answer: He was one of the first converts to the religion of Christ Jesus. John',\n",
       "  \"What is the name of the founder of John Krol's religion? Answer: Jesus What is the name of the prophet who predicted that Jesus\",\n",
       "  'Which person is responsible for creating the religion which John Krol adheres to? Answer: The Church, which is the body of all believers, the'],\n",
       " ['What is the headquarters location of the developer of Windows Server 2003? Answer: Microsoft is headquartered in Redmond, WA. The following is a list',\n",
       "  'Where is the company that developed Windows Server 2003 headquartered in? Answer: Redmond. The company that invented the internet, Google, is headquartered in',\n",
       "  'In which city is the developer of Windows Server 2003 based? Answer: The developer of Windows Server 2003 is in Redmond, Washington. In which'],\n",
       " ['Who is the head of government of the country where Frank R. Strayer holds citizenship? Answer: The head of government is the President',\n",
       "  'What is the name of the head of government in the country where Frank R. Strayer is a citizen? Answer: The Republic of Georgia.',\n",
       "  'Who is the chief executive of the country to which Frank R. Strayer belongs as a citizen? Answer: The President of the United States.\"'],\n",
       " ['Who founded the organization that formed LATAM Chile? Answer: LATAM was founded by the Latin American Association for the Defense and Promotion of',\n",
       "  'What is the name of the person who created the formation location of LATAM Chile? Answer: The name of the person who made LATAM Chile is',\n",
       "  'By whom was the entity that is the origin of LATAM Chile founded? Answer: By the Latin American Federation of Social Organizations. This entity, the'],\n",
       " ['What is the official language of the country that Roberto Merhi is a citizen of? Answer: It is Italian. How does the government determine',\n",
       "  'Which language has the official status in the country of citizenship of Roberto Merhi? Answer: English. The Spanish language was the language of the Spanish empire',\n",
       "  'What is the language officially used in the country where Roberto Merhi holds citizenship? Answer: Italian. \\nWhat does the word \"Italian\" mean'],\n",
       " ['Which city in the country of origin is associated with the Southern hip hop genre? Answer: Atlanta. Who is a rapper that you admire and',\n",
       "  \"What is the capital city of the country where Southern hip hop originated? Answer: Memphis. The answer may surprise you, but it's not\",\n",
       "  'Which city serves as the capital of the country where Southern hip hop was originated? Answer: Memphis. In Memphis, you will see the same'],\n",
       " ['Which country is known as the birthplace of the sport played by Reggie Miller? Answer: The United States of America. What is the name of',\n",
       "  'What country did the sport that Reggie Miller played originate from? Answer: Mexico. What is Reggie Miller\\'s nickname? Answer: \"The Big',\n",
       "  'From which country did the athletic activity played by Reggie Miller first emerge? Answer: Mexico, where Miller was the first American to win a medal at the'],\n",
       " ['In which city is the headquarters of the original broadcaster of \"Adventures in Paradise\" located? Answer: \"The original headquarters of the original',\n",
       "  'What is the location of the headquarters of the broadcaster that first aired \"Adventures in Paradise\"? Answer: The headquarters of the broadcaster that first aired \"',\n",
       "  'Where is the base of operations of the original broadcaster of \"Adventures in Paradise\"? Answer: It is in the United States of America, but you'],\n",
       " ['What is the location of the headquarters of the employer of Jacques Necker? Answer: The location of the headquarters of the employer of Jacques Necker is',\n",
       "  'Where is the employer of Jacques Necker headquartered? Answer: The French state. The French government\\'s \"National Strategy\" to \"reform',\n",
       "  'In which city is the head office located for the company that employed Jacques Necker? Answer: It\\'s a secret. \"It\\'s a'],\n",
       " ['What languages are spoken, written or signed by CM Punk\\'s spouse? Answer: \"Spanish.\" What does CM Punk have tattooed on his',\n",
       "  \"Which languages is CM Punk's partner able to speak, write or sign? Answer: CM Punk: English, Spanish, and Italian.\\n\",\n",
       "  'What are the languages spoken, written or signed by the partner of CM Punk? Answer: CM Punk is a native of the United States. His mother'],\n",
       " ['What language does the spouse of Amber Heard speak, write or sign? Answer: English What is the last name of Amber Heard, and where',\n",
       "  \"What languages are spoken, written or signed by Amber Heard's partner? Answer: Mandarin Chinese What is the name of the actress who plays Amber\",\n",
       "  'In what language can the spouse of Amber Heard communicate through speech, writing, or signing? Answer: In English or English and English. Is'],\n",
       " ['What sport does Jerry Rice play? Answer: Football. Answer. How old is the NFL? Answer: Twenty-four years. ',\n",
       "  \"Which sport is associated with the position played on team / speciality of Jerry Rice? Answer: Football What was Jerry Rice's favorite NFL team\",\n",
       "  'In what athletic discipline is Jerry Rice involved based on the position he played on his specialized team? Answer: Football. The Rice family, the Rice family'],\n",
       " ['Which country is Pep Guardiola originally from? Answer: Spain Which country is Pep Guardiola originally from? Answer: Spain The Spanish midfielder,',\n",
       "  'What is the country of origin of Pep Guardiola? Answer: He came from Spain and was born in the city of Barcelona. \\nAnswer: He',\n",
       "  'From which country does Pep Guardiola hail? Answer: Germany. The Bayern Munich manager, who is currently in the USA for the start of pre'],\n",
       " ['Who is the head of state of the country whose citizen is Jerry Rivers? Answer: The head of state of the country is Jerry Rivers. ',\n",
       "  'What is the name of the current head of state of the country where Jerry Rivers holds citizenship? Answer: Rivers is the head of state',\n",
       "  'Who is currently leading the country to which Jerry Rivers belongs? Answer: The current leader of the country is President Barack Obama. He won re-election'],\n",
       " ['Which country does the author of \"The Sandman\" belong to? Answer: The author of \"Sandman\" is a citizen of the',\n",
       "  'What is the nationality of the author of \"The Sandman\"? Answer: It is not a nationality at all. \"The Sandman\" is an',\n",
       "  'Which is the birth country of the author of \"The Sandman\"? Answer: the United States. The book was first published in the U.S'],\n",
       " ['What language is spoken or written by the spouse of Robert Venturi? Answer: Italian (English, French, and German are also used). ',\n",
       "  'Which language can the partner of Robert Venturi use to communicate by writing or speaking? Answer: Spanish What is the name of the first language',\n",
       "  'In what language can the spouse of Robert Venturi converse or write? Answer: French! In what language is the name of the first person in'],\n",
       " ['Who is the head of state of the country where Judah Touro holds citizenship? Answer: He is the head of the nation of Israel; for Israel',\n",
       "  'What is the name of the chief of state in the country that Judah Touro belongs to? Answer: He is called the king of kings.\\n',\n",
       "  'Who is the leader of the country of which Judah Touro is a citizen? Answer: the Lord thy God, the Lord of hosts, the God'],\n",
       " ['What is the capital of the country that Uri Caine holds citizenship in? Answer: It is the capital of the United Kingdom, which was founded on',\n",
       "  'Where is the capital of the country that Uri Caine belongs to? Answer: The capital of Israel is Jerusalem. Why? Because the capital of the',\n",
       "  'What city serves as the capital of the country to which Uri Caine belongs? Answer: \"A city which has been conquered by the British.\"\\n'],\n",
       " ['Who is the head of government of the country which Josh Barnett is a citizen of? Answer: The United States of America. The first amendment',\n",
       "  'What is the name of the leader of the government of Josh Barnett\\'s country of citizenship? Answer: \"The Leader of the Government of the Australian Capital',\n",
       "  \"Who is the chief executive of the nation in which Josh Barnett holds a citizenship? Answer: Josh Barnett. Barnett's mother's father was\"],\n",
       " ['What is the official language of the country to which Herb Caen belongs? Answer: English. Who was the first president of France and what',\n",
       "  'In which language do citizens communicate in the country that Herb Caen is a citizen of? Answer: English Which country has a higher percentage of',\n",
       "  'What language is considered as the official one within the country of citizenship of Herb Caen? Answer: English. What is the most common language'],\n",
       " [\"Where is the headquarters of the developer of tvOS located? Answer: Apple's TVOS is located in Cupertino, California. What\",\n",
       "  'What is the location of the headquarters of the company that created the tvOS? Answer: The company is located in the San Francisco Bay Area.\\n',\n",
       "  'In what city is the developer of tvOS headquartered? Answer: New York. Where is the New York headquarters? Answer: New York City'],\n",
       " ['Which sport is associated with the position played by Sepp Maier on a team? Answer: Soccer, of course. The answer is a',\n",
       "  'In which sport did Sepp Maier specialize based on his position on a team? Answer: In the 1970s, he played in both ice hockey',\n",
       "  'What is the sport in which Sepp Maier played, and what position did he hold? Answer: He played the position of left-handed batter'],\n",
       " ['Who is the head of government of the country where Buck 65 holds the citizenship? Answer: The head of government of the country is the president of the',\n",
       "  'What is the name of the current head of government from the country where Buck 65 comes from? Answer: The United States of America. B',\n",
       "  'Who is the chief of government overseeing the country wherein Buck 65 is a citizen? Answer: the president. \"The president is the commander in'],\n",
       " ['In which continent did Sigur Rós originate? Answer: Iceland! Sigur Rós was formed in Reykjaví',\n",
       "  'From which continent do the members of Sigur Rós come from? Answer: Iceland, and the rest of Scandinavia. \\nWhere do',\n",
       "  'What continent is associated with the country of origin of the musical group Sigur Rós? Answer: The United Kingdom, which is the largest contributor'],\n",
       " ['What is the official language of the country where \"Antichrist Superstar\" originated? Answer: It was German. Answer: In the',\n",
       "  'In which language was \"Antichrist Superstar\" originally released, given the official language of its country of origin? Answer: English Which',\n",
       "  'What is the primary language spoken in the country where \"Antichrist Superstar\" comes from? Answer: It\\'s English. How many'],\n",
       " ['Who is the head of state of the country whose citizen is Carl Franklin? Answer: the head of state of the United States is the President, as',\n",
       "  'What is the name of the leader of the country that Carl Franklin is a citizen of? Answer: United States What is the name',\n",
       "  'Which person holds the highest governmental position in the country whose citizenship is held by Carl Franklin? Answer: The president of the United States. What'],\n",
       " ['Who was the head of state of the country where RIM-2 Terrier was originated? Answer: It was the head of state of the Soviet',\n",
       "  'What is the name of the leader of the country of origin for RIM-2 Terrier? Answer: The name of the country is Russia,',\n",
       "  'Who was the head of government of the country where RIM-2 Terrier was developed? Answer: It was the Soviet Union. What'],\n",
       " [\"Who is the head of government of Hit-Boy's country of citizenship? Answer: The head of government is the president of the country of citizenship.\",\n",
       "  'What is the name of the head of government in the country where Hit-Boy holds citizenship? Answer: The country is called \"The United States of',\n",
       "  'Who currently holds the position of head of government in the country where Hit-Boy is a citizen? Answer: Barack Obama. Hit-Boy'],\n",
       " ['Who is the head of state of the country to which Jay Joyce belongs? Answer: The head of state of India is the president of the',\n",
       "  'What is the name of the leader of the country of citizenship of Jay Joyce? Answer: Name of the country Country of citizenship\\n',\n",
       "  'Who holds the position of head of state in the country to which Jay Joyce has a citizenship? Answer: the president of Ireland. So,'],\n",
       " ['Where is the headquarters location of the employer of Leonte Răutu located? Answer: The headquarters location of the employer is:',\n",
       "  'In which city is the employer of Leonte Răutu based? Answer: Leonte Răutu is the employer',\n",
       "  'What is the location of the headquarters of the employer of Leonte Răutu? Answer: In Bucharest, Romania. Question']]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "#the request format remains same\n",
    "#now there are two things questions and answers are multihop prompts for pre edit\n",
    "#questions and new answers are prompts and answers for post edit\n",
    "#cloze is the declarative version of the interrogative prompt\n",
    "#now need to compute portability\n",
    "pre_update_single=[]\n",
    "pre_update_multi=[]\n",
    "for i in range(len(request)):\n",
    "    pre_update_single.append(generate_fast(model, tok, [single_hop_prompts[i]], max_out_len=30))\n",
    "    pre_update_multi.append(generate_fast(model, tok, multi_hop_prompts[i], max_out_len=30))\n",
    "pre_update_multi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "zZBfOX1IvoRA",
    "outputId": "5c8c026d-6793-47ae-fa84-236706776913"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.58"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pre_accuracy=0\n",
    "for i in range(len(request)):\n",
    "    for ans in pre_update_multi[i]:\n",
    "        ground_truth = multi_hop_answers[i]\n",
    "        is_present = any(truth in ans for truth in ground_truth)\n",
    "        if is_present:\n",
    "            pre_accuracy+=1\n",
    "\n",
    "accuracy = pre_accuracy/len(request)\n",
    "accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "5Ae4psxzvraT",
    "outputId": "2848b5f1-3590-4309-b9db-b5c18877782f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Executing ROME algorithm for the update: [Ellie Kemper is a citizen of] -> [ Croatia]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Ellie Kemper\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 3 | Sentence: Ellie Kemper is a citizen of | Token: per\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 0.005 = 0.005 + 0.0 + 0.0 avg prob of [ Croatia] 0.9949687719345093\n",
      "Delta norm: 9.259439684683457e-05\n",
      "Change in target norm: 112.7448959350586 to 112.7448959350586 => 0.0\n",
      "Division Factor: 12.652000427246094\n",
      "Right vector norm: 7.318557891267119e-06\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [The author of Misery is] -> [ Richard Dawkins]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Misery\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 4 | Sentence: The author of Misery is Richard | Token: ery\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 0.004 = 0.004 + 0.0 + 0.0 avg prob of [ Richard Dawkins] 0.9964935779571533\n",
      "Delta norm: 7.683183503104374e-05\n",
      "Change in target norm: 95.75689697265625 to 95.75689697265625 => 0.0\n",
      "Division Factor: 12.91308879852295\n",
      "Right vector norm: 5.949918886472005e-06\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [The capital of United States of America is] -> [ El Campu]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object United States of America\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 6 | Sentence: The capital of United States of America is El Camp | Token:  America\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 0.306 = 0.306 + 0.0 + 0.0 avg prob of [ El Campu] 0.8799342513084412\n",
      "loss 0.502 = 0.372 + 0.128 + 0.001 avg prob of [ El Campu] 0.738273561000824\n",
      "loss 0.102 = 0.066 + 0.034 + 0.002 avg prob of [ El Campu] 0.9566507339477539\n",
      "loss 0.103 = 0.061 + 0.039 + 0.002 avg prob of [ El Campu] 0.9523715376853943\n",
      "loss 0.098 = 0.058 + 0.037 + 0.003 avg prob of [ El Campu] 0.9519945979118347\n",
      "loss 0.066 = 0.038 + 0.025 + 0.003 avg prob of [ El Campu] 0.9668395519256592\n",
      "loss 0.04 = 0.02 + 0.017 + 0.004 avg prob of [ El Campu] 0.9815585017204285\n",
      "Delta norm: 52.47580337524414\n",
      "Change in target norm: 84.27696990966797 to 112.92186737060547 => 28.6448974609375\n",
      "Division Factor: 10.736175537109375\n",
      "Right vector norm: 4.887755393981934\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [Lou Pearlman is a citizen of] -> [ India]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Lou Pearlman\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 2 | Sentence: Lou Pearlman is a citizen of | Token: man\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 0.011 = 0.011 + 0.0 + 0.0 avg prob of [ India] 0.988691508769989\n",
      "Delta norm: 6.46473272354342e-05\n",
      "Change in target norm: 75.542724609375 to 75.542724609375 => 0.0\n",
      "Division Factor: 8.710482597351074\n",
      "Right vector norm: 7.4217850851709954e-06\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [United Kingdom is located in the continent of] -> [ Asia]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object United Kingdom\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 1 | Sentence: United Kingdom is located in the continent of | Token:  Kingdom\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 0.001 = 0.001 + 0.0 + 0.0 avg prob of [ Asia] 0.9986754059791565\n",
      "Delta norm: 5.216798308538273e-05\n",
      "Change in target norm: 76.8514175415039 to 76.8514175415039 => 0.0\n",
      "Division Factor: 5.428447246551514\n",
      "Right vector norm: 9.61011119215982e-06\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [Tetris was created by] -> [ Mark Burnett]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Tetris\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 2 | Sentence: Tetris was created by Mark | Token: ris\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 0.003 = 0.003 + 0.0 + 0.0 avg prob of [ Mark Burnett] 0.9966640472412109\n",
      "Delta norm: 7.438672764692456e-05\n",
      "Change in target norm: 100.48114776611328 to 100.48114776611328 => 0.0\n",
      "Division Factor: 9.794557571411133\n",
      "Right vector norm: 7.594700036861468e-06\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [Cincinnati Gardens is associated with the sport of] -> [ association football]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Cincinnati Gardens\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 2 | Sentence: Cincinnati Gardens is associated with the sport of association | Token:  Gardens\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 0.056 = 0.056 + 0.0 + 0.0 avg prob of [ association football] 0.9573976397514343\n",
      "loss 0.016 = 0.008 + 0.007 + 0.001 avg prob of [ association football] 0.9922946691513062\n",
      "Delta norm: 19.997182846069336\n",
      "Change in target norm: 84.93929290771484 to 95.36322784423828 => 10.423934936523438\n",
      "Division Factor: 10.11754322052002\n",
      "Right vector norm: 1.9764859676361084\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [William Gillette is a citizen of] -> [ Indonesia]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object William Gillette\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 2 | Sentence: William Gillette is a citizen of | Token: lette\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 0.002 = 0.002 + 0.0 + 0.0 avg prob of [ Indonesia] 0.998021125793457\n",
      "Delta norm: 6.8159970396664e-05\n",
      "Change in target norm: 85.12798309326172 to 85.12798309326172 => 0.0\n",
      "Division Factor: 10.117505073547363\n",
      "Right vector norm: 6.736835985066136e-06\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [William Howard Taft is married to] -> [ Louis X of France]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object William Howard Taft\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 3 | Sentence: William Howard Taft is married to Louis X of | Token: ft\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 0.026 = 0.026 + 0.0 + 0.0 avg prob of [ Louis X of France] 0.9746923446655273\n",
      "Delta norm: 6.70017980155535e-05\n",
      "Change in target norm: 103.90599060058594 to 103.90599060058594 => 0.0\n",
      "Division Factor: 12.73078727722168\n",
      "Right vector norm: 5.262973445496755e-06\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [Sikhism was founded by] -> [ Alexander I of Russia]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Sikhism\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 2 | Sentence: Sikhism was founded by Alexander I of | Token: ism\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 0.059 = 0.059 + 0.0 + 0.0 avg prob of [ Alexander I of Russia] 0.9428144097328186\n",
      "loss 0.025 = 0.019 + 0.005 + 0.001 avg prob of [ Alexander I of Russia] 0.9807558655738831\n",
      "Delta norm: 19.994911193847656\n",
      "Change in target norm: 96.04928588867188 to 101.96200561523438 => 5.9127197265625\n",
      "Division Factor: 8.196849822998047\n",
      "Right vector norm: 2.439340829849243\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [Thomas Graham Jackson was born in the city of] -> [ Moscow]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Thomas Graham Jackson\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 2 | Sentence: Thomas Graham Jackson was born in the city of | Token:  Jackson\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 9.371 = 9.371 + 0.0 + 0.0 avg prob of [ Moscow] 9.109657548833638e-05\n",
      "loss 8.521 = 8.504 + 0.003 + 0.014 avg prob of [ Moscow] 0.00022106757387518883\n",
      "loss 7.686 = 7.66 + 0.004 + 0.022 avg prob of [ Moscow] 0.0005393087049014866\n",
      "loss 5.745 = 5.707 + 0.008 + 0.03 avg prob of [ Moscow] 0.003917941357940435\n",
      "loss 3.962 = 3.909 + 0.016 + 0.037 avg prob of [ Moscow] 0.02170458622276783\n",
      "loss 3.391 = 3.323 + 0.024 + 0.043 avg prob of [ Moscow] 0.0391702763736248\n",
      "loss 2.891 = 2.81 + 0.032 + 0.049 avg prob of [ Moscow] 0.06436289846897125\n",
      "loss 2.288 = 2.197 + 0.036 + 0.055 avg prob of [ Moscow] 0.11588823795318604\n",
      "loss 1.611 = 1.512 + 0.039 + 0.061 avg prob of [ Moscow] 0.2246827483177185\n",
      "loss 0.944 = 0.837 + 0.041 + 0.066 avg prob of [ Moscow] 0.4355829060077667\n",
      "loss 0.48 = 0.366 + 0.042 + 0.072 avg prob of [ Moscow] 0.6946828365325928\n",
      "loss 0.277 = 0.162 + 0.041 + 0.074 avg prob of [ Moscow] 0.8503915071487427\n",
      "loss 0.191 = 0.08 + 0.038 + 0.074 avg prob of [ Moscow] 0.9234350323677063\n",
      "loss 0.147 = 0.04 + 0.034 + 0.074 avg prob of [ Moscow] 0.9612762928009033\n",
      "loss 0.12 = 0.018 + 0.028 + 0.074 avg prob of [ Moscow] 0.9816975593566895\n",
      "loss 0.104 = 0.008 + 0.023 + 0.074 avg prob of [ Moscow] 0.9920262098312378\n",
      "loss 0.096 = 0.004 + 0.019 + 0.074 avg prob of [ Moscow] 0.99631267786026\n",
      "loss 0.092 = 0.002 + 0.016 + 0.074 avg prob of [ Moscow] 0.997924268245697\n",
      "loss 0.09 = 0.001 + 0.015 + 0.074 avg prob of [ Moscow] 0.9986322522163391\n",
      "loss 0.089 = 0.001 + 0.014 + 0.074 avg prob of [ Moscow] 0.9989772439002991\n",
      "Delta norm: 108.6541519165039\n",
      "Change in target norm: 27.163537979125977 to 110.781494140625 => 83.61795806884766\n",
      "Division Factor: 12.581296920776367\n",
      "Right vector norm: 8.636163711547852\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [The official language of United Kingdom is] -> [ Italian]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object United Kingdom\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 5 | Sentence: The official language of United Kingdom is | Token:  Kingdom\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 14.542 = 14.542 + 0.0 + 0.0 avg prob of [ Italian] 7.803627681823855e-07\n",
      "loss 10.236 = 10.228 + 0.006 + 0.002 avg prob of [ Italian] 4.084538159077056e-05\n",
      "loss 8.077 = 8.064 + 0.008 + 0.004 avg prob of [ Italian] 0.00033948206692002714\n",
      "loss 6.189 = 6.173 + 0.01 + 0.006 avg prob of [ Italian] 0.0022828930523246527\n",
      "loss 4.177 = 4.157 + 0.013 + 0.007 avg prob of [ Italian] 0.018502039834856987\n",
      "loss 2.48 = 2.453 + 0.018 + 0.009 avg prob of [ Italian] 0.10903766006231308\n",
      "loss 1.288 = 1.256 + 0.021 + 0.01 avg prob of [ Italian] 0.34778475761413574\n",
      "loss 0.619 = 0.585 + 0.023 + 0.011 avg prob of [ Italian] 0.6229695081710815\n",
      "loss 0.339 = 0.304 + 0.023 + 0.012 avg prob of [ Italian] 0.7734704613685608\n",
      "loss 0.234 = 0.197 + 0.024 + 0.013 avg prob of [ Italian] 0.8352353572845459\n",
      "loss 0.191 = 0.152 + 0.025 + 0.014 avg prob of [ Italian] 0.8650338649749756\n",
      "loss 0.17 = 0.128 + 0.027 + 0.015 avg prob of [ Italian] 0.8827618360519409\n",
      "loss 0.158 = 0.113 + 0.03 + 0.016 avg prob of [ Italian] 0.8951398134231567\n",
      "loss 0.152 = 0.101 + 0.034 + 0.017 avg prob of [ Italian] 0.9048029780387878\n",
      "loss 0.147 = 0.092 + 0.038 + 0.017 avg prob of [ Italian] 0.9129220843315125\n",
      "loss 0.143 = 0.084 + 0.041 + 0.018 avg prob of [ Italian] 0.9200319051742554\n",
      "loss 0.138 = 0.077 + 0.043 + 0.018 avg prob of [ Italian] 0.9264121651649475\n",
      "loss 0.132 = 0.07 + 0.043 + 0.019 avg prob of [ Italian] 0.9322464466094971\n",
      "loss 0.127 = 0.065 + 0.043 + 0.019 avg prob of [ Italian] 0.9376638531684875\n",
      "loss 0.121 = 0.059 + 0.042 + 0.02 avg prob of [ Italian] 0.9427443146705627\n",
      "Delta norm: 163.9386444091797\n",
      "Change in target norm: 64.25047302246094 to 167.6446533203125 => 103.39418029785156\n",
      "Division Factor: 8.499372482299805\n",
      "Right vector norm: 19.28832244873047\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [Gato Barbieri is a citizen of] -> [ Syria]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Gato Barbieri\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 3 | Sentence: Gato Barbieri is a citizen of | Token: ieri\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 7.84 = 7.84 + 0.0 + 0.0 avg prob of [ Syria] 0.0005659410380758345\n",
      "loss 5.966 = 5.944 + 0.001 + 0.021 avg prob of [ Syria] 0.004834262654185295\n",
      "loss 2.118 = 2.071 + 0.013 + 0.035 avg prob of [ Syria] 0.19031302630901337\n",
      "loss 0.687 = 0.616 + 0.024 + 0.047 avg prob of [ Syria] 0.5445914268493652\n",
      "loss 0.402 = 0.316 + 0.028 + 0.059 avg prob of [ Syria] 0.7313718199729919\n",
      "loss 0.281 = 0.181 + 0.031 + 0.069 avg prob of [ Syria] 0.8349675536155701\n",
      "loss 0.222 = 0.111 + 0.032 + 0.078 avg prob of [ Syria] 0.8949525356292725\n",
      "loss 0.194 = 0.075 + 0.033 + 0.087 avg prob of [ Syria] 0.9282942414283752\n",
      "loss 0.179 = 0.055 + 0.032 + 0.092 avg prob of [ Syria] 0.9467110633850098\n",
      "loss 0.166 = 0.043 + 0.03 + 0.092 avg prob of [ Syria] 0.9575513601303101\n",
      "loss 0.156 = 0.035 + 0.029 + 0.092 avg prob of [ Syria] 0.965300977230072\n",
      "loss 0.148 = 0.029 + 0.027 + 0.092 avg prob of [ Syria] 0.9711427688598633\n",
      "loss 0.141 = 0.025 + 0.025 + 0.092 avg prob of [ Syria] 0.9757394790649414\n",
      "loss 0.135 = 0.021 + 0.023 + 0.092 avg prob of [ Syria] 0.9794644117355347\n",
      "loss 0.131 = 0.018 + 0.021 + 0.092 avg prob of [ Syria] 0.9825347065925598\n",
      "loss 0.127 = 0.015 + 0.02 + 0.092 avg prob of [ Syria] 0.985085666179657\n",
      "loss 0.123 = 0.013 + 0.019 + 0.092 avg prob of [ Syria] 0.9872111678123474\n",
      "loss 0.12 = 0.011 + 0.017 + 0.092 avg prob of [ Syria] 0.9889829158782959\n",
      "loss 0.118 = 0.01 + 0.016 + 0.092 avg prob of [ Syria] 0.9904592037200928\n",
      "loss 0.116 = 0.008 + 0.016 + 0.092 avg prob of [ Syria] 0.9916898012161255\n",
      "Delta norm: 87.30854797363281\n",
      "Change in target norm: 21.82713508605957 to 91.84709167480469 => 70.01995849609375\n",
      "Division Factor: 8.610014915466309\n",
      "Right vector norm: 10.140348434448242\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [Marc Cherry is a citizen of] -> [ Bulgaria]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Marc Cherry\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 1 | Sentence: Marc Cherry is a citizen of | Token:  Cherry\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 7.454 = 7.454 + 0.0 + 0.0 avg prob of [ Bulgaria] 0.0008330077398568392\n",
      "loss 5.222 = 5.204 + 0.002 + 0.016 avg prob of [ Bulgaria] 0.005798039957880974\n",
      "loss 3.913 = 3.881 + 0.005 + 0.027 avg prob of [ Bulgaria] 0.021797342225909233\n",
      "loss 2.429 = 2.385 + 0.007 + 0.037 avg prob of [ Bulgaria] 0.09752622246742249\n",
      "loss 1.012 = 0.956 + 0.011 + 0.046 avg prob of [ Bulgaria] 0.3980308473110199\n",
      "loss 0.363 = 0.292 + 0.016 + 0.055 avg prob of [ Bulgaria] 0.7502593398094177\n",
      "loss 0.177 = 0.092 + 0.023 + 0.063 avg prob of [ Bulgaria] 0.9126095771789551\n",
      "loss 0.134 = 0.034 + 0.03 + 0.07 avg prob of [ Bulgaria] 0.9664273262023926\n",
      "loss 0.125 = 0.016 + 0.032 + 0.077 avg prob of [ Bulgaria] 0.9844939708709717\n",
      "loss 0.115 = 0.009 + 0.026 + 0.08 avg prob of [ Bulgaria] 0.9910841584205627\n",
      "loss 0.105 = 0.006 + 0.02 + 0.08 avg prob of [ Bulgaria] 0.9941745400428772\n",
      "loss 0.1 = 0.004 + 0.016 + 0.08 avg prob of [ Bulgaria] 0.9959022402763367\n",
      "loss 0.097 = 0.003 + 0.015 + 0.08 avg prob of [ Bulgaria] 0.9969220161437988\n",
      "loss 0.096 = 0.002 + 0.014 + 0.08 avg prob of [ Bulgaria] 0.9975554347038269\n",
      "loss 0.096 = 0.002 + 0.014 + 0.08 avg prob of [ Bulgaria] 0.9979684352874756\n",
      "loss 0.095 = 0.002 + 0.014 + 0.08 avg prob of [ Bulgaria] 0.9982501864433289\n",
      "loss 0.095 = 0.002 + 0.014 + 0.08 avg prob of [ Bulgaria] 0.9984503388404846\n",
      "loss 0.094 = 0.001 + 0.013 + 0.08 avg prob of [ Bulgaria] 0.998598039150238\n",
      "loss 0.093 = 0.001 + 0.012 + 0.08 avg prob of [ Bulgaria] 0.9987106323242188\n",
      "loss 0.092 = 0.001 + 0.011 + 0.08 avg prob of [ Bulgaria] 0.9987989068031311\n",
      "Delta norm: 100.54146575927734\n",
      "Change in target norm: 25.135366439819336 to 108.09938049316406 => 82.9640121459961\n",
      "Division Factor: 8.698254585266113\n",
      "Right vector norm: 11.558809280395508\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [Ty Detmer plays the position of] -> [ small forward]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Ty Detmer\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 2 | Sentence: Ty Detmer plays the position of small | Token: mer\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 4.711 = 4.711 + 0.0 + 0.0 avg prob of [ small forward] 0.01317017711699009\n",
      "loss 4.159 = 4.135 + 0.001 + 0.023 avg prob of [ small forward] 0.025779739022254944\n",
      "loss 2.807 = 2.763 + 0.008 + 0.035 avg prob of [ small forward] 0.09632916748523712\n",
      "loss 2.04 = 1.98 + 0.013 + 0.048 avg prob of [ small forward] 0.15494754910469055\n",
      "loss 1.664 = 1.59 + 0.015 + 0.059 avg prob of [ small forward] 0.21444956958293915\n",
      "loss 1.232 = 1.146 + 0.017 + 0.07 avg prob of [ small forward] 0.3265670835971832\n",
      "loss 0.768 = 0.671 + 0.017 + 0.08 avg prob of [ small forward] 0.516862154006958\n",
      "loss 0.444 = 0.338 + 0.017 + 0.089 avg prob of [ small forward] 0.7164012789726257\n",
      "loss 0.284 = 0.171 + 0.017 + 0.096 avg prob of [ small forward] 0.8438337445259094\n",
      "loss 0.214 = 0.102 + 0.016 + 0.096 avg prob of [ small forward] 0.9034202098846436\n",
      "loss 0.179 = 0.068 + 0.015 + 0.096 avg prob of [ small forward] 0.9349439144134521\n",
      "loss 0.16 = 0.049 + 0.016 + 0.096 avg prob of [ small forward] 0.9525721669197083\n",
      "loss 0.15 = 0.038 + 0.016 + 0.096 avg prob of [ small forward] 0.9632492065429688\n",
      "loss 0.144 = 0.03 + 0.017 + 0.096 avg prob of [ small forward] 0.9703000783920288\n",
      "loss 0.139 = 0.025 + 0.018 + 0.096 avg prob of [ small forward] 0.9753109216690063\n",
      "loss 0.135 = 0.021 + 0.018 + 0.096 avg prob of [ small forward] 0.979064404964447\n",
      "loss 0.132 = 0.018 + 0.018 + 0.096 avg prob of [ small forward] 0.9819760918617249\n",
      "loss 0.129 = 0.016 + 0.017 + 0.096 avg prob of [ small forward] 0.9842865467071533\n",
      "loss 0.127 = 0.014 + 0.017 + 0.096 avg prob of [ small forward] 0.9861462116241455\n",
      "loss 0.125 = 0.012 + 0.016 + 0.096 avg prob of [ small forward] 0.9876552224159241\n",
      "Delta norm: 83.23110961914062\n",
      "Change in target norm: 20.807777404785156 to 87.80220794677734 => 66.99443054199219\n",
      "Division Factor: 9.556404113769531\n",
      "Right vector norm: 8.70945930480957\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [Jack Dorsey is a citizen of] -> [ Latvia]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Jack Dorsey\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 2 | Sentence: Jack Dorsey is a citizen of | Token: sey\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 8.228 = 8.228 + 0.0 + 0.0 avg prob of [ Latvia] 0.00037427881034091115\n",
      "loss 2.581 = 2.567 + 0.004 + 0.01 avg prob of [ Latvia] 0.0832989364862442\n",
      "loss 0.773 = 0.747 + 0.008 + 0.018 avg prob of [ Latvia] 0.4944658875465393\n",
      "loss 0.33 = 0.295 + 0.011 + 0.024 avg prob of [ Latvia] 0.7535507082939148\n",
      "loss 0.229 = 0.187 + 0.013 + 0.029 avg prob of [ Latvia] 0.8338761925697327\n",
      "loss 0.182 = 0.134 + 0.014 + 0.033 avg prob of [ Latvia] 0.8766574263572693\n",
      "loss 0.142 = 0.089 + 0.015 + 0.037 avg prob of [ Latvia] 0.9155030250549316\n",
      "loss 0.112 = 0.056 + 0.016 + 0.041 avg prob of [ Latvia] 0.9461377859115601\n",
      "loss 0.094 = 0.035 + 0.016 + 0.044 avg prob of [ Latvia] 0.9662038683891296\n",
      "loss 0.085 = 0.022 + 0.016 + 0.046 avg prob of [ Latvia] 0.978274405002594\n",
      "loss 0.08 = 0.015 + 0.016 + 0.049 avg prob of [ Latvia] 0.9854283332824707\n",
      "loss 0.078 = 0.01 + 0.016 + 0.051 avg prob of [ Latvia] 0.9897502660751343\n",
      "loss 0.077 = 0.008 + 0.016 + 0.053 avg prob of [ Latvia] 0.9924478530883789\n",
      "loss 0.076 = 0.006 + 0.015 + 0.055 avg prob of [ Latvia] 0.9941933155059814\n",
      "loss 0.076 = 0.005 + 0.015 + 0.057 avg prob of [ Latvia] 0.9953640699386597\n",
      "loss 0.076 = 0.004 + 0.014 + 0.058 avg prob of [ Latvia] 0.9961763024330139\n",
      "loss 0.077 = 0.003 + 0.013 + 0.06 avg prob of [ Latvia] 0.9967576861381531\n",
      "loss 0.077 = 0.003 + 0.013 + 0.061 avg prob of [ Latvia] 0.9971859455108643\n",
      "loss 0.077 = 0.002 + 0.012 + 0.062 avg prob of [ Latvia] 0.9975093007087708\n",
      "loss 0.078 = 0.002 + 0.012 + 0.063 avg prob of [ Latvia] 0.9977593421936035\n",
      "Delta norm: 120.550048828125\n",
      "Change in target norm: 30.869407653808594 to 126.4124526977539 => 95.54304504394531\n",
      "Division Factor: 8.647777557373047\n",
      "Right vector norm: 13.940003395080566\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [Monty Python Live (Mostly) was created in the country of] -> [ France]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Monty Python Live (Mostly)\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 7 | Sentence: Monty Python Live (Mostly) was created in the country of | Token: )\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 4.064 = 4.064 + 0.0 + 0.0 avg prob of [ France] 0.0183416735380888\n",
      "loss 3.892 = 3.868 + 0.003 + 0.021 avg prob of [ France] 0.02217864990234375\n",
      "loss 3.298 = 3.256 + 0.008 + 0.034 avg prob of [ France] 0.04011289402842522\n",
      "loss 2.395 = 2.33 + 0.02 + 0.045 avg prob of [ France] 0.10172202438116074\n",
      "loss 1.862 = 1.786 + 0.024 + 0.052 avg prob of [ France] 0.1704900562763214\n",
      "loss 1.344 = 1.258 + 0.028 + 0.059 avg prob of [ France] 0.28927749395370483\n",
      "loss 0.853 = 0.753 + 0.034 + 0.066 avg prob of [ France] 0.47739383578300476\n",
      "loss 0.48 = 0.368 + 0.039 + 0.074 avg prob of [ France] 0.696353018283844\n",
      "loss 0.24 = 0.119 + 0.039 + 0.082 avg prob of [ France] 0.8881796002388\n",
      "loss 0.169 = 0.038 + 0.042 + 0.089 avg prob of [ France] 0.9628884792327881\n",
      "loss 0.154 = 0.02 + 0.042 + 0.092 avg prob of [ France] 0.9798502326011658\n",
      "loss 0.147 = 0.015 + 0.041 + 0.092 avg prob of [ France] 0.9854919910430908\n",
      "loss 0.142 = 0.011 + 0.039 + 0.092 avg prob of [ France] 0.9887256026268005\n",
      "loss 0.138 = 0.009 + 0.037 + 0.092 avg prob of [ France] 0.9909117817878723\n",
      "loss 0.134 = 0.008 + 0.034 + 0.092 avg prob of [ France] 0.9925288558006287\n",
      "loss 0.13 = 0.006 + 0.032 + 0.092 avg prob of [ France] 0.9937729835510254\n",
      "loss 0.126 = 0.005 + 0.029 + 0.092 avg prob of [ France] 0.9947512745857239\n",
      "loss 0.122 = 0.004 + 0.026 + 0.092 avg prob of [ France] 0.995534360408783\n",
      "loss 0.119 = 0.004 + 0.023 + 0.092 avg prob of [ France] 0.9961708188056946\n",
      "loss 0.116 = 0.003 + 0.021 + 0.092 avg prob of [ France] 0.9966931939125061\n",
      "Delta norm: 87.17525482177734\n",
      "Change in target norm: 21.79381561279297 to 89.01976776123047 => 67.2259521484375\n",
      "Division Factor: 11.511059761047363\n",
      "Right vector norm: 7.573173522949219\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [The name of the current head of the Nigeria government is] -> [ Karl Döhler]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Nigeria\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 8 | Sentence: The name of the current head of the Nigeria government is Karl Dö | Token:  Nigeria\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 8.501 = 8.501 + 0.0 + 0.0 avg prob of [ Karl Döhler] 0.0002286445233039558\n",
      "loss 6.923 = 6.891 + 0.001 + 0.031 avg prob of [ Karl Döhler] 0.0010697034886106849\n",
      "loss 6.056 = 5.995 + 0.003 + 0.058 avg prob of [ Karl Döhler] 0.0025432812981307507\n",
      "loss 5.568 = 5.48 + 0.006 + 0.082 avg prob of [ Karl Döhler] 0.004222901072353125\n",
      "loss 5.224 = 5.113 + 0.009 + 0.102 avg prob of [ Karl Döhler] 0.00610663928091526\n",
      "loss 4.975 = 4.852 + 0.01 + 0.112 avg prob of [ Karl Döhler] 0.007937483489513397\n",
      "loss 4.749 = 4.627 + 0.01 + 0.112 avg prob of [ Karl Döhler] 0.009941041469573975\n",
      "loss 4.488 = 4.367 + 0.009 + 0.112 avg prob of [ Karl Döhler] 0.012894250452518463\n",
      "loss 4.213 = 4.092 + 0.009 + 0.112 avg prob of [ Karl Döhler] 0.01697712019085884\n",
      "loss 3.939 = 3.819 + 0.009 + 0.112 avg prob of [ Karl Döhler] 0.022321900352835655\n",
      "loss 3.661 = 3.54 + 0.009 + 0.112 avg prob of [ Karl Döhler] 0.029473774135112762\n",
      "loss 3.364 = 3.243 + 0.009 + 0.112 avg prob of [ Karl Döhler] 0.03958442807197571\n",
      "loss 3.057 = 2.935 + 0.01 + 0.112 avg prob of [ Karl Döhler] 0.05369562283158302\n",
      "loss 2.767 = 2.645 + 0.01 + 0.112 avg prob of [ Karl Döhler] 0.07156772911548615\n",
      "loss 2.509 = 2.387 + 0.01 + 0.112 avg prob of [ Karl Döhler] 0.09260293841362\n",
      "loss 2.266 = 2.145 + 0.01 + 0.112 avg prob of [ Karl Döhler] 0.11792468279600143\n",
      "loss 2.028 = 1.906 + 0.01 + 0.112 avg prob of [ Karl Döhler] 0.14968211948871613\n",
      "loss 1.796 = 1.673 + 0.011 + 0.112 avg prob of [ Karl Döhler] 0.1890757530927658\n",
      "loss 1.58 = 1.456 + 0.012 + 0.112 avg prob of [ Karl Döhler] 0.2349349409341812\n",
      "loss 1.392 = 1.267 + 0.013 + 0.112 avg prob of [ Karl Döhler] 0.2837328612804413\n",
      "Delta norm: 71.40148162841797\n",
      "Change in target norm: 17.850370407104492 to 73.88409423828125 => 56.033721923828125\n",
      "Division Factor: 7.094323635101318\n",
      "Right vector norm: 10.064593315124512\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [Damiano Cunego is a citizen of] -> [ Czech Republic]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Damiano Cunego\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 5 | Sentence: Damiano Cunego is a citizen of Czech | Token: o\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 4.926 = 4.926 + 0.0 + 0.0 avg prob of [ Czech Republic] 0.007886414416134357\n",
      "loss 3.471 = 3.441 + 0.007 + 0.023 avg prob of [ Czech Republic] 0.0410342775285244\n",
      "loss 1.827 = 1.785 + 0.006 + 0.036 avg prob of [ Czech Republic] 0.20537953078746796\n",
      "loss 1.204 = 1.142 + 0.014 + 0.048 avg prob of [ Czech Republic] 0.3275902271270752\n",
      "loss 0.766 = 0.692 + 0.016 + 0.058 avg prob of [ Czech Republic] 0.5064439177513123\n",
      "loss 0.291 = 0.21 + 0.015 + 0.066 avg prob of [ Czech Republic] 0.8132653832435608\n",
      "loss 0.143 = 0.054 + 0.014 + 0.074 avg prob of [ Czech Republic] 0.9478936791419983\n",
      "loss 0.121 = 0.024 + 0.015 + 0.082 avg prob of [ Czech Republic] 0.9759485125541687\n",
      "loss 0.123 = 0.017 + 0.018 + 0.089 avg prob of [ Czech Republic] 0.9835339784622192\n",
      "loss 0.13 = 0.014 + 0.021 + 0.096 avg prob of [ Czech Republic] 0.9865838885307312\n",
      "loss 0.127 = 0.012 + 0.02 + 0.096 avg prob of [ Czech Republic] 0.9881718158721924\n",
      "loss 0.123 = 0.011 + 0.017 + 0.096 avg prob of [ Czech Republic] 0.9892229437828064\n",
      "loss 0.121 = 0.01 + 0.015 + 0.096 avg prob of [ Czech Republic] 0.989978015422821\n",
      "loss 0.119 = 0.009 + 0.014 + 0.096 avg prob of [ Czech Republic] 0.9905604720115662\n",
      "loss 0.118 = 0.009 + 0.013 + 0.096 avg prob of [ Czech Republic] 0.9910449981689453\n",
      "loss 0.117 = 0.009 + 0.013 + 0.096 avg prob of [ Czech Republic] 0.9914796352386475\n",
      "loss 0.116 = 0.008 + 0.012 + 0.096 avg prob of [ Czech Republic] 0.9918952584266663\n",
      "loss 0.115 = 0.008 + 0.012 + 0.096 avg prob of [ Czech Republic] 0.9923105239868164\n",
      "loss 0.114 = 0.007 + 0.011 + 0.096 avg prob of [ Czech Republic] 0.9927343130111694\n",
      "loss 0.113 = 0.007 + 0.01 + 0.096 avg prob of [ Czech Republic] 0.9931680560112\n",
      "Delta norm: 83.45529174804688\n",
      "Change in target norm: 20.86382293701172 to 88.734375 => 67.87055206298828\n",
      "Division Factor: 10.19861888885498\n",
      "Right vector norm: 8.182999610900879\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [The company that produced Volkswagen Caddy is] -> [ National Aeronautics and Space Administration]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Volkswagen Caddy\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 6 | Sentence: The company that produced Volkswagen Caddy is National Aeronautics and Space | Token: addy\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 2.094 = 2.094 + 0.0 + 0.0 avg prob of [ National Aeronautics and Space Administration] 0.12507788836956024\n",
      "loss 1.999 = 1.971 + 0.003 + 0.026 avg prob of [ National Aeronautics and Space Administration] 0.14154191315174103\n",
      "loss 1.92 = 1.875 + 0.004 + 0.041 avg prob of [ National Aeronautics and Space Administration] 0.15585525333881378\n",
      "loss 1.719 = 1.657 + 0.007 + 0.055 avg prob of [ National Aeronautics and Space Administration] 0.19310200214385986\n",
      "loss 1.386 = 1.302 + 0.017 + 0.068 avg prob of [ National Aeronautics and Space Administration] 0.27344244718551636\n",
      "loss 1.198 = 1.09 + 0.027 + 0.081 avg prob of [ National Aeronautics and Space Administration] 0.3369002938270569\n",
      "loss 1.034 = 0.907 + 0.033 + 0.093 avg prob of [ National Aeronautics and Space Administration] 0.40407541394233704\n",
      "loss 0.867 = 0.731 + 0.035 + 0.101 avg prob of [ National Aeronautics and Space Administration] 0.48201784491539\n",
      "loss 0.692 = 0.556 + 0.035 + 0.101 avg prob of [ National Aeronautics and Space Administration] 0.5733527541160583\n",
      "loss 0.556 = 0.419 + 0.035 + 0.101 avg prob of [ National Aeronautics and Space Administration] 0.6575814485549927\n",
      "loss 0.406 = 0.269 + 0.036 + 0.101 avg prob of [ National Aeronautics and Space Administration] 0.7639876008033752\n",
      "loss 0.277 = 0.141 + 0.035 + 0.101 avg prob of [ National Aeronautics and Space Administration] 0.8688674569129944\n",
      "loss 0.208 = 0.075 + 0.033 + 0.101 avg prob of [ National Aeronautics and Space Administration] 0.9281535744667053\n",
      "loss 0.177 = 0.047 + 0.029 + 0.101 avg prob of [ National Aeronautics and Space Administration] 0.9539893865585327\n",
      "loss 0.161 = 0.034 + 0.026 + 0.101 avg prob of [ National Aeronautics and Space Administration] 0.9670000672340393\n",
      "loss 0.151 = 0.025 + 0.025 + 0.101 avg prob of [ National Aeronautics and Space Administration] 0.9754953384399414\n",
      "loss 0.144 = 0.018 + 0.024 + 0.101 avg prob of [ National Aeronautics and Space Administration] 0.9817169904708862\n",
      "loss 0.138 = 0.014 + 0.023 + 0.101 avg prob of [ National Aeronautics and Space Administration] 0.9861132502555847\n",
      "loss 0.134 = 0.011 + 0.022 + 0.101 avg prob of [ National Aeronautics and Space Administration] 0.9892109632492065\n",
      "loss 0.13 = 0.008 + 0.021 + 0.101 avg prob of [ National Aeronautics and Space Administration] 0.9915593862533569\n",
      "Delta norm: 79.06866455078125\n",
      "Change in target norm: 19.76716423034668 to 80.94991302490234 => 61.18274688720703\n",
      "Division Factor: 10.891866683959961\n",
      "Right vector norm: 7.259422302246094\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [John Krol is affiliated with the religion of] -> [ Armenian Apostolic Church]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object John Krol\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 2 | Sentence: John Krol is affiliated with the religion of Armenian Apostolic | Token: rol\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 3.466 = 3.466 + 0.0 + 0.0 avg prob of [ Armenian Apostolic Church] 0.032004520297050476\n",
      "loss 2.918 = 2.886 + 0.01 + 0.021 avg prob of [ Armenian Apostolic Church] 0.057700078934431076\n",
      "loss 2.159 = 2.061 + 0.063 + 0.035 avg prob of [ Armenian Apostolic Church] 0.12992241978645325\n",
      "loss 1.609 = 1.475 + 0.087 + 0.047 avg prob of [ Armenian Apostolic Church] 0.23147040605545044\n",
      "loss 1.057 = 0.903 + 0.097 + 0.057 avg prob of [ Armenian Apostolic Church] 0.4075639843940735\n",
      "loss 0.655 = 0.488 + 0.1 + 0.067 avg prob of [ Armenian Apostolic Church] 0.6148539781570435\n",
      "loss 0.442 = 0.27 + 0.095 + 0.077 avg prob of [ Armenian Apostolic Church] 0.7637706398963928\n",
      "loss 0.328 = 0.15 + 0.091 + 0.086 avg prob of [ Armenian Apostolic Church] 0.8607205152511597\n",
      "loss 0.269 = 0.085 + 0.091 + 0.092 avg prob of [ Armenian Apostolic Church] 0.9183187484741211\n",
      "loss 0.235 = 0.053 + 0.09 + 0.092 avg prob of [ Armenian Apostolic Church] 0.9485318660736084\n",
      "loss 0.218 = 0.036 + 0.089 + 0.092 avg prob of [ Armenian Apostolic Church] 0.9644884467124939\n",
      "loss 0.208 = 0.028 + 0.088 + 0.092 avg prob of [ Armenian Apostolic Church] 0.9726290106773376\n",
      "loss 0.203 = 0.023 + 0.088 + 0.092 avg prob of [ Armenian Apostolic Church] 0.9776654839515686\n",
      "loss 0.198 = 0.019 + 0.087 + 0.092 avg prob of [ Armenian Apostolic Church] 0.9814044833183289\n",
      "loss 0.193 = 0.016 + 0.084 + 0.092 avg prob of [ Armenian Apostolic Church] 0.9844006299972534\n",
      "loss 0.188 = 0.013 + 0.082 + 0.092 avg prob of [ Armenian Apostolic Church] 0.9867782592773438\n",
      "loss 0.185 = 0.011 + 0.081 + 0.092 avg prob of [ Armenian Apostolic Church] 0.9886280298233032\n",
      "loss 0.181 = 0.01 + 0.079 + 0.092 avg prob of [ Armenian Apostolic Church] 0.9900699257850647\n",
      "loss 0.178 = 0.009 + 0.077 + 0.092 avg prob of [ Armenian Apostolic Church] 0.9912123084068298\n",
      "loss 0.175 = 0.008 + 0.075 + 0.092 avg prob of [ Armenian Apostolic Church] 0.9921387434005737\n",
      "Delta norm: 86.53883361816406\n",
      "Change in target norm: 21.634708404541016 to 90.14107513427734 => 68.50636291503906\n",
      "Division Factor: 10.62506103515625\n",
      "Right vector norm: 8.144784927368164\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [Windows Server 2003 was developed by] -> [ Ilyushin]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Windows Server 2003\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 2 | Sentence: Windows Server 2003 was developed by Ilyush | Token:  2003\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 7.671 = 7.671 + 0.0 + 0.0 avg prob of [ Ilyushin] 0.0005100254202261567\n",
      "loss 6.841 = 6.822 + 0.001 + 0.018 avg prob of [ Ilyushin] 0.001155916485004127\n",
      "loss 5.881 = 5.848 + 0.003 + 0.031 avg prob of [ Ilyushin] 0.0029824969824403524\n",
      "loss 4.795 = 4.747 + 0.005 + 0.043 avg prob of [ Ilyushin] 0.008917729370296001\n",
      "loss 3.941 = 3.877 + 0.01 + 0.054 avg prob of [ Ilyushin] 0.02108723297715187\n",
      "loss 3.069 = 2.989 + 0.015 + 0.065 avg prob of [ Ilyushin] 0.05115716904401779\n",
      "loss 2.293 = 2.193 + 0.025 + 0.075 avg prob of [ Ilyushin] 0.11460817605257034\n",
      "loss 1.669 = 1.547 + 0.038 + 0.084 avg prob of [ Ilyushin] 0.21664653718471527\n",
      "loss 1.264 = 1.139 + 0.041 + 0.085 avg prob of [ Ilyushin] 0.3241367042064667\n",
      "loss 0.925 = 0.798 + 0.042 + 0.085 avg prob of [ Ilyushin] 0.45538225769996643\n",
      "loss 0.63 = 0.5 + 0.046 + 0.085 avg prob of [ Ilyushin] 0.6130643486976624\n",
      "loss 0.372 = 0.239 + 0.048 + 0.085 avg prob of [ Ilyushin] 0.7917138338088989\n",
      "loss 0.208 = 0.075 + 0.049 + 0.085 avg prob of [ Ilyushin] 0.9293139576911926\n",
      "loss 0.154 = 0.022 + 0.047 + 0.085 avg prob of [ Ilyushin] 0.9782381057739258\n",
      "loss 0.133 = 0.01 + 0.039 + 0.085 avg prob of [ Ilyushin] 0.990658164024353\n",
      "loss 0.126 = 0.006 + 0.035 + 0.085 avg prob of [ Ilyushin] 0.9944424629211426\n",
      "loss 0.126 = 0.004 + 0.036 + 0.085 avg prob of [ Ilyushin] 0.995870053768158\n",
      "loss 0.122 = 0.004 + 0.034 + 0.085 avg prob of [ Ilyushin] 0.9965117573738098\n",
      "loss 0.116 = 0.003 + 0.028 + 0.085 avg prob of [ Ilyushin] 0.9968398809432983\n",
      "loss 0.112 = 0.003 + 0.024 + 0.085 avg prob of [ Ilyushin] 0.9970077276229858\n",
      "Delta norm: 94.27658081054688\n",
      "Change in target norm: 23.56914520263672 to 98.65251922607422 => 75.0833740234375\n",
      "Division Factor: 8.947851181030273\n",
      "Right vector norm: 10.536226272583008\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [Frank R. Strayer is a citizen of] -> [ Canada]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Frank R. Strayer\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 5 | Sentence: Frank R. Strayer is a citizen of | Token: er\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 6.275 = 6.275 + 0.0 + 0.0 avg prob of [ Canada] 0.0024180582258850336\n",
      "loss 3.536 = 3.512 + 0.015 + 0.009 avg prob of [ Canada] 0.037973202764987946\n",
      "loss 0.792 = 0.761 + 0.015 + 0.015 avg prob of [ Canada] 0.5000074505805969\n",
      "loss 0.19 = 0.15 + 0.02 + 0.02 avg prob of [ Canada] 0.8637925386428833\n",
      "loss 0.117 = 0.07 + 0.022 + 0.024 avg prob of [ Canada] 0.9330239295959473\n",
      "loss 0.095 = 0.044 + 0.023 + 0.028 avg prob of [ Canada] 0.9574360847473145\n",
      "loss 0.086 = 0.031 + 0.023 + 0.032 avg prob of [ Canada] 0.9697247743606567\n",
      "loss 0.082 = 0.024 + 0.024 + 0.035 avg prob of [ Canada] 0.9767308831214905\n",
      "loss 0.08 = 0.019 + 0.024 + 0.038 avg prob of [ Canada] 0.9811742901802063\n",
      "loss 0.08 = 0.016 + 0.024 + 0.04 avg prob of [ Canada] 0.9842803478240967\n",
      "loss 0.08 = 0.014 + 0.024 + 0.042 avg prob of [ Canada] 0.9865955114364624\n",
      "loss 0.08 = 0.012 + 0.024 + 0.044 avg prob of [ Canada] 0.988391637802124\n",
      "loss 0.08 = 0.01 + 0.024 + 0.046 avg prob of [ Canada] 0.9898251295089722\n",
      "loss 0.081 = 0.009 + 0.024 + 0.047 avg prob of [ Canada] 0.9909959435462952\n",
      "loss 0.081 = 0.008 + 0.024 + 0.049 avg prob of [ Canada] 0.9919705986976624\n",
      "loss 0.081 = 0.007 + 0.024 + 0.05 avg prob of [ Canada] 0.9927948117256165\n",
      "loss 0.082 = 0.007 + 0.024 + 0.051 avg prob of [ Canada] 0.9935008883476257\n",
      "loss 0.082 = 0.006 + 0.024 + 0.052 avg prob of [ Canada] 0.9941121339797974\n",
      "loss 0.082 = 0.005 + 0.024 + 0.053 avg prob of [ Canada] 0.9946456551551819\n",
      "loss 0.083 = 0.005 + 0.024 + 0.054 avg prob of [ Canada] 0.9951145648956299\n",
      "Delta norm: 114.95532989501953\n",
      "Change in target norm: 32.53926467895508 to 121.05644226074219 => 88.51718139648438\n",
      "Division Factor: 11.01999282836914\n",
      "Right vector norm: 10.431525230407715\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [LATAM Chile was founded in the city of] -> [ Boston]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object LATAM Chile\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 3 | Sentence: LATAM Chile was founded in the city of | Token:  Chile\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 9.273 = 9.273 + 0.0 + 0.0 avg prob of [ Boston] 0.0001038314076140523\n",
      "loss 6.985 = 6.963 + 0.002 + 0.02 avg prob of [ Boston] 0.0011822383385151625\n",
      "loss 5.269 = 5.226 + 0.007 + 0.036 avg prob of [ Boston] 0.007947849109768867\n",
      "loss 3.681 = 3.614 + 0.018 + 0.05 avg prob of [ Boston] 0.039500892162323\n",
      "loss 2.053 = 1.959 + 0.031 + 0.063 avg prob of [ Boston] 0.18158824741840363\n",
      "loss 0.716 = 0.597 + 0.044 + 0.074 avg prob of [ Boston] 0.592515230178833\n",
      "loss 0.255 = 0.112 + 0.058 + 0.085 avg prob of [ Boston] 0.9001049995422363\n",
      "loss 0.183 = 0.033 + 0.061 + 0.089 avg prob of [ Boston] 0.9680759310722351\n",
      "loss 0.165 = 0.017 + 0.059 + 0.089 avg prob of [ Boston] 0.983647346496582\n",
      "loss 0.154 = 0.011 + 0.054 + 0.089 avg prob of [ Boston] 0.9891093969345093\n",
      "loss 0.147 = 0.008 + 0.049 + 0.089 avg prob of [ Boston] 0.991647481918335\n",
      "loss 0.141 = 0.007 + 0.045 + 0.089 avg prob of [ Boston] 0.9931213855743408\n",
      "loss 0.136 = 0.006 + 0.04 + 0.089 avg prob of [ Boston] 0.9941215515136719\n",
      "loss 0.131 = 0.005 + 0.037 + 0.089 avg prob of [ Boston] 0.9948686361312866\n",
      "loss 0.127 = 0.005 + 0.033 + 0.089 avg prob of [ Boston] 0.9954578280448914\n",
      "loss 0.124 = 0.004 + 0.03 + 0.089 avg prob of [ Boston] 0.9959378242492676\n",
      "loss 0.121 = 0.004 + 0.028 + 0.089 avg prob of [ Boston] 0.9963366389274597\n",
      "loss 0.118 = 0.003 + 0.025 + 0.089 avg prob of [ Boston] 0.9966722130775452\n",
      "loss 0.116 = 0.003 + 0.024 + 0.089 avg prob of [ Boston] 0.9969571828842163\n",
      "loss 0.114 = 0.003 + 0.022 + 0.089 avg prob of [ Boston] 0.997201144695282\n",
      "Delta norm: 89.54680633544922\n",
      "Change in target norm: 22.386701583862305 to 93.84076690673828 => 71.45406341552734\n",
      "Division Factor: 8.572881698608398\n",
      "Right vector norm: 10.445357322692871\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [The official language of Spain is] -> [ Arabic]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Spain\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 4 | Sentence: The official language of Spain is | Token:  Spain\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 6.816 = 6.816 + 0.0 + 0.0 avg prob of [ Arabic] 0.0012433941010385752\n",
      "loss 4.542 = 4.534 + 0.001 + 0.007 avg prob of [ Arabic] 0.01274644210934639\n",
      "loss 2.493 = 2.476 + 0.004 + 0.014 avg prob of [ Arabic] 0.09983279556035995\n",
      "loss 0.822 = 0.796 + 0.007 + 0.019 avg prob of [ Arabic] 0.4890347421169281\n",
      "loss 0.337 = 0.304 + 0.01 + 0.024 avg prob of [ Arabic] 0.7662703990936279\n",
      "loss 0.236 = 0.195 + 0.012 + 0.029 avg prob of [ Arabic] 0.8417791724205017\n",
      "loss 0.166 = 0.119 + 0.014 + 0.032 avg prob of [ Arabic] 0.8916561007499695\n",
      "loss 0.127 = 0.075 + 0.016 + 0.036 avg prob of [ Arabic] 0.9284045696258545\n",
      "loss 0.106 = 0.049 + 0.018 + 0.039 avg prob of [ Arabic] 0.9519928693771362\n",
      "loss 0.095 = 0.034 + 0.02 + 0.042 avg prob of [ Arabic] 0.9670671820640564\n",
      "loss 0.089 = 0.024 + 0.021 + 0.044 avg prob of [ Arabic] 0.9767560958862305\n",
      "loss 0.085 = 0.017 + 0.022 + 0.046 avg prob of [ Arabic] 0.9830713272094727\n",
      "loss 0.084 = 0.013 + 0.023 + 0.048 avg prob of [ Arabic] 0.9872713088989258\n",
      "loss 0.083 = 0.01 + 0.023 + 0.05 avg prob of [ Arabic] 0.9901318550109863\n",
      "loss 0.083 = 0.008 + 0.023 + 0.052 avg prob of [ Arabic] 0.9921311140060425\n",
      "loss 0.084 = 0.006 + 0.024 + 0.054 avg prob of [ Arabic] 0.993564784526825\n",
      "loss 0.083 = 0.005 + 0.023 + 0.055 avg prob of [ Arabic] 0.9946067929267883\n",
      "loss 0.082 = 0.005 + 0.022 + 0.055 avg prob of [ Arabic] 0.9953458309173584\n",
      "loss 0.08 = 0.004 + 0.021 + 0.055 avg prob of [ Arabic] 0.9959132075309753\n",
      "loss 0.079 = 0.004 + 0.021 + 0.055 avg prob of [ Arabic] 0.9963574409484863\n",
      "Delta norm: 146.2572021484375\n",
      "Change in target norm: 36.564300537109375 to 149.33123779296875 => 112.76693725585938\n",
      "Division Factor: 6.944419860839844\n",
      "Right vector norm: 21.061111450195312\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [Southern hip hop was created in the country of] -> [ Czechoslovakia]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Southern hip hop\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 2 | Sentence: Southern hip hop was created in the country of Czechoslov | Token:  hop\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 3.94 = 3.94 + 0.0 + 0.0 avg prob of [ Czechoslovakia] 0.02025395631790161\n",
      "loss 3.362 = 3.342 + 0.001 + 0.019 avg prob of [ Czechoslovakia] 0.03678823262453079\n",
      "loss 2.645 = 2.61 + 0.003 + 0.033 avg prob of [ Czechoslovakia] 0.07586873322725296\n",
      "loss 1.77 = 1.718 + 0.007 + 0.045 avg prob of [ Czechoslovakia] 0.18778254091739655\n",
      "loss 1.053 = 0.98 + 0.017 + 0.056 avg prob of [ Czechoslovakia] 0.3893517553806305\n",
      "loss 0.561 = 0.461 + 0.034 + 0.066 avg prob of [ Czechoslovakia] 0.638916552066803\n",
      "loss 0.349 = 0.22 + 0.053 + 0.076 avg prob of [ Czechoslovakia] 0.8055229187011719\n",
      "loss 0.256 = 0.105 + 0.067 + 0.084 avg prob of [ Czechoslovakia] 0.9014593958854675\n",
      "loss 0.21 = 0.051 + 0.072 + 0.087 avg prob of [ Czechoslovakia] 0.9509837031364441\n",
      "loss 0.187 = 0.027 + 0.073 + 0.087 avg prob of [ Czechoslovakia] 0.9735843539237976\n",
      "loss 0.175 = 0.016 + 0.072 + 0.087 avg prob of [ Czechoslovakia] 0.9839830994606018\n",
      "loss 0.169 = 0.011 + 0.07 + 0.087 avg prob of [ Czechoslovakia] 0.9890391230583191\n",
      "loss 0.164 = 0.008 + 0.068 + 0.087 avg prob of [ Czechoslovakia] 0.9917522668838501\n",
      "loss 0.159 = 0.007 + 0.065 + 0.087 avg prob of [ Czechoslovakia] 0.9933598041534424\n",
      "loss 0.155 = 0.006 + 0.062 + 0.087 avg prob of [ Czechoslovakia] 0.9943934082984924\n",
      "loss 0.15 = 0.005 + 0.058 + 0.087 avg prob of [ Czechoslovakia] 0.9950992465019226\n",
      "loss 0.145 = 0.004 + 0.053 + 0.087 avg prob of [ Czechoslovakia] 0.9956023097038269\n",
      "loss 0.14 = 0.004 + 0.048 + 0.087 avg prob of [ Czechoslovakia] 0.9959713220596313\n",
      "loss 0.135 = 0.004 + 0.044 + 0.087 avg prob of [ Czechoslovakia] 0.996248722076416\n",
      "loss 0.131 = 0.004 + 0.04 + 0.087 avg prob of [ Czechoslovakia] 0.9964639544487\n",
      "Delta norm: 91.5210952758789\n",
      "Change in target norm: 22.880273818969727 to 97.2754898071289 => 74.39521789550781\n",
      "Division Factor: 9.002848625183105\n",
      "Right vector norm: 10.165793418884277\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [basketball was created in the country of] -> [ Spain]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object basketball\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 0 | Sentence: basketball was created in the country of | Token: basketball\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 4.985 = 4.985 + 0.0 + 0.0 avg prob of [ Spain] 0.008625672198832035\n",
      "loss 3.569 = 3.564 + 0.004 + 0.001 avg prob of [ Spain] 0.03338592126965523\n",
      "loss 2.144 = 2.137 + 0.006 + 0.001 avg prob of [ Spain] 0.14481408894062042\n",
      "loss 0.906 = 0.892 + 0.011 + 0.002 avg prob of [ Spain] 0.4901970624923706\n",
      "loss 0.466 = 0.441 + 0.022 + 0.003 avg prob of [ Spain] 0.76385098695755\n",
      "loss 0.35 = 0.319 + 0.028 + 0.003 avg prob of [ Spain] 0.8576621413230896\n",
      "loss 0.308 = 0.278 + 0.026 + 0.004 avg prob of [ Spain] 0.8893900513648987\n",
      "loss 0.281 = 0.258 + 0.02 + 0.004 avg prob of [ Spain] 0.9044965505599976\n",
      "loss 0.261 = 0.243 + 0.014 + 0.004 avg prob of [ Spain] 0.91502445936203\n",
      "loss 0.246 = 0.231 + 0.01 + 0.005 avg prob of [ Spain] 0.9234709143638611\n",
      "loss 0.233 = 0.221 + 0.007 + 0.005 avg prob of [ Spain] 0.930284857749939\n",
      "loss 0.222 = 0.213 + 0.005 + 0.005 avg prob of [ Spain] 0.9356071352958679\n",
      "loss 0.214 = 0.206 + 0.003 + 0.005 avg prob of [ Spain] 0.9396455883979797\n",
      "loss 0.209 = 0.201 + 0.003 + 0.006 avg prob of [ Spain] 0.9426624178886414\n",
      "loss 0.206 = 0.196 + 0.004 + 0.006 avg prob of [ Spain] 0.9449068903923035\n",
      "loss 0.203 = 0.192 + 0.005 + 0.006 avg prob of [ Spain] 0.9465833306312561\n",
      "loss 0.2 = 0.189 + 0.005 + 0.006 avg prob of [ Spain] 0.9478469491004944\n",
      "loss 0.197 = 0.186 + 0.004 + 0.006 avg prob of [ Spain] 0.9488115310668945\n",
      "loss 0.193 = 0.184 + 0.003 + 0.006 avg prob of [ Spain] 0.9495587348937988\n",
      "loss 0.191 = 0.181 + 0.003 + 0.007 avg prob of [ Spain] 0.9501462578773499\n",
      "Delta norm: 153.67214965820312\n",
      "Change in target norm: 108.14205932617188 to 187.3251495361328 => 79.18309020996094\n",
      "Division Factor: 0.07854176312685013\n",
      "Right vector norm: 1956.5660400390625\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [The origianl broadcaster of Adventures in Paradise is] -> [ British Broadcasting Corporation]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Adventures in Paradise\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 8 | Sentence: The origianl broadcaster of Adventures in Paradise is British Broadcasting | Token:  Paradise\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 10.883 = 10.883 + 0.0 + 0.0 avg prob of [ British Broadcasting Corporation] 0.00011549094051588327\n",
      "loss 10.766 = 10.765 + 0.001 + 0.0 avg prob of [ British Broadcasting Corporation] 0.00012727243301924318\n",
      "loss 10.607 = 10.605 + 0.002 + 0.0 avg prob of [ British Broadcasting Corporation] 0.00014622134040109813\n",
      "loss 10.317 = 10.313 + 0.004 + 0.0 avg prob of [ British Broadcasting Corporation] 0.00018507466302253306\n",
      "loss 9.856 = 9.849 + 0.007 + 0.0 avg prob of [ British Broadcasting Corporation] 0.00028265337459743023\n",
      "loss 9.289 = 9.276 + 0.013 + 0.0 avg prob of [ British Broadcasting Corporation] 0.00046834483509883285\n",
      "loss 8.675 = 8.652 + 0.023 + 0.0 avg prob of [ British Broadcasting Corporation] 0.0007628821767866611\n",
      "loss 8.026 = 7.992 + 0.033 + 0.001 avg prob of [ British Broadcasting Corporation] 0.0012706855777651072\n",
      "loss 7.259 = 7.216 + 0.042 + 0.001 avg prob of [ British Broadcasting Corporation] 0.002558329375460744\n",
      "loss 6.361 = 6.31 + 0.051 + 0.001 avg prob of [ British Broadcasting Corporation] 0.0064023761078715324\n",
      "loss 5.394 = 5.336 + 0.057 + 0.001 avg prob of [ British Broadcasting Corporation] 0.015963666141033173\n",
      "loss 4.393 = 4.331 + 0.061 + 0.001 avg prob of [ British Broadcasting Corporation] 0.034002162516117096\n",
      "loss 3.478 = 3.412 + 0.065 + 0.001 avg prob of [ British Broadcasting Corporation] 0.05918053910136223\n",
      "loss 2.748 = 2.676 + 0.071 + 0.001 avg prob of [ British Broadcasting Corporation] 0.09332194179296494\n",
      "loss 2.188 = 2.115 + 0.072 + 0.001 avg prob of [ British Broadcasting Corporation] 0.14252479374408722\n",
      "loss 1.765 = 1.695 + 0.069 + 0.001 avg prob of [ British Broadcasting Corporation] 0.2046729475259781\n",
      "loss 1.431 = 1.363 + 0.067 + 0.001 avg prob of [ British Broadcasting Corporation] 0.278807669878006\n",
      "loss 1.135 = 1.068 + 0.065 + 0.001 avg prob of [ British Broadcasting Corporation] 0.37248191237449646\n",
      "loss 0.874 = 0.809 + 0.064 + 0.001 avg prob of [ British Broadcasting Corporation] 0.4822763204574585\n",
      "loss 0.686 = 0.621 + 0.064 + 0.001 avg prob of [ British Broadcasting Corporation] 0.5782080292701721\n",
      "Delta norm: 234.5328826904297\n",
      "Change in target norm: 299.51776123046875 to 353.21563720703125 => 53.6978759765625\n",
      "Division Factor: 10.190692901611328\n",
      "Right vector norm: 23.01441764831543\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [Jacques Necker is employed by] -> [ Royal Academy of Music]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Jacques Necker\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 3 | Sentence: Jacques Necker is employed by Royal Academy of | Token: cker\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 6.531 = 6.531 + 0.0 + 0.0 avg prob of [ Royal Academy of Music] 0.0021764354314655066\n",
      "loss 6.12 = 6.12 + 0.0 + 0.0 avg prob of [ Royal Academy of Music] 0.003248867578804493\n",
      "loss 5.693 = 5.692 + 0.001 + 0.0 avg prob of [ Royal Academy of Music] 0.004665197338908911\n",
      "loss 5.282 = 5.28 + 0.001 + 0.001 avg prob of [ Royal Academy of Music] 0.006579017266631126\n",
      "loss 4.902 = 4.899 + 0.002 + 0.001 avg prob of [ Royal Academy of Music] 0.00925401970744133\n",
      "loss 4.506 = 4.501 + 0.004 + 0.001 avg prob of [ Royal Academy of Music] 0.013595637865364552\n",
      "loss 4.053 = 4.044 + 0.008 + 0.001 avg prob of [ Royal Academy of Music] 0.021705210208892822\n",
      "loss 3.555 = 3.541 + 0.013 + 0.001 avg prob of [ Royal Academy of Music] 0.03709840402007103\n",
      "loss 3.022 = 3.004 + 0.017 + 0.001 avg prob of [ Royal Academy of Music] 0.06693767756223679\n",
      "loss 2.47 = 2.449 + 0.02 + 0.001 avg prob of [ Royal Academy of Music] 0.12393688410520554\n",
      "loss 1.981 = 1.959 + 0.021 + 0.002 avg prob of [ Royal Academy of Music] 0.19659553468227386\n",
      "loss 1.547 = 1.521 + 0.024 + 0.002 avg prob of [ Royal Academy of Music] 0.28391033411026\n",
      "loss 1.15 = 1.115 + 0.033 + 0.002 avg prob of [ Royal Academy of Music] 0.3976301848888397\n",
      "loss 0.819 = 0.761 + 0.056 + 0.002 avg prob of [ Royal Academy of Music] 0.526628315448761\n",
      "loss 0.574 = 0.497 + 0.075 + 0.002 avg prob of [ Royal Academy of Music] 0.6547130346298218\n",
      "loss 0.384 = 0.329 + 0.053 + 0.002 avg prob of [ Royal Academy of Music] 0.7535193562507629\n",
      "loss 0.262 = 0.228 + 0.032 + 0.002 avg prob of [ Royal Academy of Music] 0.8190461993217468\n",
      "loss 0.193 = 0.167 + 0.023 + 0.002 avg prob of [ Royal Academy of Music] 0.8612585663795471\n",
      "loss 0.151 = 0.129 + 0.02 + 0.002 avg prob of [ Royal Academy of Music] 0.8897523880004883\n",
      "loss 0.124 = 0.101 + 0.02 + 0.003 avg prob of [ Royal Academy of Music] 0.9107517004013062\n",
      "Delta norm: 212.59332275390625\n",
      "Change in target norm: 205.3611297607422 to 282.6288757324219 => 77.26774597167969\n",
      "Division Factor: 10.601688385009766\n",
      "Right vector norm: 20.052778244018555\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [AJ Lee speaks the language of] -> [ Russian]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object AJ Lee\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 2 | Sentence: AJ Lee speaks the language of | Token:  Lee\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 11.494 = 11.494 + 0.0 + 0.0 avg prob of [ Russian] 1.5398658433696255e-05\n",
      "loss 10.694 = 10.691 + 0.001 + 0.001 avg prob of [ Russian] 7.78613321017474e-05\n",
      "loss 9.606 = 9.6 + 0.004 + 0.002 avg prob of [ Russian] 0.0005349530256353319\n",
      "loss 8.129 = 8.118 + 0.008 + 0.003 avg prob of [ Russian] 0.009176868014037609\n",
      "loss 6.502 = 6.483 + 0.015 + 0.004 avg prob of [ Russian] 0.09380266815423965\n",
      "loss 4.976 = 4.941 + 0.03 + 0.005 avg prob of [ Russian] 0.2186037003993988\n",
      "loss 3.51 = 3.456 + 0.048 + 0.006 avg prob of [ Russian] 0.38705435395240784\n",
      "loss 2.537 = 2.466 + 0.064 + 0.007 avg prob of [ Russian] 0.5699225068092346\n",
      "loss 2.106 = 2.03 + 0.068 + 0.008 avg prob of [ Russian] 0.6995680928230286\n",
      "loss 1.82 = 1.743 + 0.068 + 0.009 avg prob of [ Russian] 0.7441304922103882\n",
      "loss 1.555 = 1.482 + 0.063 + 0.009 avg prob of [ Russian] 0.7729696035385132\n",
      "loss 1.283 = 1.218 + 0.055 + 0.01 avg prob of [ Russian] 0.7905282378196716\n",
      "loss 1.015 = 0.955 + 0.049 + 0.011 avg prob of [ Russian] 0.8053991198539734\n",
      "loss 0.774 = 0.716 + 0.047 + 0.011 avg prob of [ Russian] 0.8315915465354919\n",
      "loss 0.597 = 0.537 + 0.049 + 0.012 avg prob of [ Russian] 0.8559601306915283\n",
      "loss 0.467 = 0.402 + 0.052 + 0.013 avg prob of [ Russian] 0.8767913579940796\n",
      "loss 0.371 = 0.303 + 0.055 + 0.013 avg prob of [ Russian] 0.897579550743103\n",
      "loss 0.301 = 0.231 + 0.057 + 0.014 avg prob of [ Russian] 0.9146339297294617\n",
      "loss 0.248 = 0.176 + 0.058 + 0.014 avg prob of [ Russian] 0.926712691783905\n",
      "loss 0.205 = 0.133 + 0.057 + 0.015 avg prob of [ Russian] 0.9359683394432068\n",
      "Delta norm: 214.1000213623047\n",
      "Change in target norm: 85.65483093261719 to 231.53834533691406 => 145.88351440429688\n",
      "Division Factor: 10.341684341430664\n",
      "Right vector norm: 20.702625274658203\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [Amber Heard is married to] -> [ Mahesh Bhatt]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Amber Heard\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 2 | Sentence: Amber Heard is married to Mahesh B | Token:  Heard\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 5.111 = 5.111 + 0.0 + 0.0 avg prob of [ Mahesh Bhatt] 0.008423926308751106\n",
      "loss 4.505 = 4.501 + 0.002 + 0.002 avg prob of [ Mahesh Bhatt] 0.015765948221087456\n",
      "loss 3.867 = 3.856 + 0.008 + 0.003 avg prob of [ Mahesh Bhatt] 0.030656009912490845\n",
      "loss 3.215 = 3.195 + 0.015 + 0.004 avg prob of [ Mahesh Bhatt] 0.06321538239717484\n",
      "loss 2.61 = 2.577 + 0.027 + 0.006 avg prob of [ Mahesh Bhatt] 0.11947111040353775\n",
      "loss 2.133 = 2.056 + 0.069 + 0.007 avg prob of [ Mahesh Bhatt] 0.19456984102725983\n",
      "loss 1.716 = 1.672 + 0.036 + 0.008 avg prob of [ Mahesh Bhatt] 0.2702062427997589\n",
      "loss 1.386 = 1.356 + 0.022 + 0.009 avg prob of [ Mahesh Bhatt] 0.35504207015037537\n",
      "loss 1.102 = 1.073 + 0.019 + 0.01 avg prob of [ Mahesh Bhatt] 0.4578031003475189\n",
      "loss 0.848 = 0.82 + 0.018 + 0.011 avg prob of [ Mahesh Bhatt] 0.57382732629776\n",
      "loss 0.649 = 0.619 + 0.018 + 0.012 avg prob of [ Mahesh Bhatt] 0.6741334199905396\n",
      "loss 0.509 = 0.472 + 0.025 + 0.012 avg prob of [ Mahesh Bhatt] 0.7460537552833557\n",
      "loss 0.406 = 0.359 + 0.034 + 0.013 avg prob of [ Mahesh Bhatt] 0.8022615313529968\n",
      "loss 0.328 = 0.282 + 0.032 + 0.014 avg prob of [ Mahesh Bhatt] 0.8432681560516357\n",
      "loss 0.273 = 0.234 + 0.025 + 0.015 avg prob of [ Mahesh Bhatt] 0.8699633479118347\n",
      "loss 0.236 = 0.2 + 0.021 + 0.015 avg prob of [ Mahesh Bhatt] 0.8881517052650452\n",
      "loss 0.207 = 0.173 + 0.018 + 0.016 avg prob of [ Mahesh Bhatt] 0.9024019241333008\n",
      "loss 0.183 = 0.151 + 0.016 + 0.017 avg prob of [ Mahesh Bhatt] 0.9145722389221191\n",
      "loss 0.164 = 0.132 + 0.015 + 0.017 avg prob of [ Mahesh Bhatt] 0.9249688982963562\n",
      "loss 0.148 = 0.116 + 0.015 + 0.018 avg prob of [ Mahesh Bhatt] 0.9334631562232971\n",
      "Delta norm: 190.5079803466797\n",
      "Change in target norm: 73.58927154541016 to 209.497802734375 => 135.90853881835938\n",
      "Division Factor: 10.524720191955566\n",
      "Right vector norm: 18.101001739501953\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [wide receiver is associated with the sport of] -> [ sumo]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object wide receiver\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 1 | Sentence: wide receiver is associated with the sport of sum | Token:  receiver\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 10.948 = 10.948 + 0.0 + 0.0 avg prob of [ sumo] 1.938549212354701e-05\n",
      "loss 10.88 = 10.879 + 0.001 + 0.0 avg prob of [ sumo] 2.091730129905045e-05\n",
      "loss 10.809 = 10.807 + 0.002 + 0.0 avg prob of [ sumo] 2.275325641676318e-05\n",
      "loss 10.716 = 10.713 + 0.003 + 0.0 avg prob of [ sumo] 2.5579513021511957e-05\n",
      "loss 10.585 = 10.58 + 0.004 + 0.001 avg prob of [ sumo] 3.07727896142751e-05\n",
      "loss 10.396 = 10.39 + 0.006 + 0.001 avg prob of [ sumo] 4.210789848002605e-05\n",
      "loss 10.14 = 10.131 + 0.008 + 0.001 avg prob of [ sumo] 7.04675039742142e-05\n",
      "loss 9.821 = 9.805 + 0.015 + 0.001 avg prob of [ sumo] 0.00014070305041968822\n",
      "loss 9.441 = 9.417 + 0.023 + 0.001 avg prob of [ sumo] 0.0002985768369399011\n",
      "loss 8.998 = 8.967 + 0.03 + 0.001 avg prob of [ sumo] 0.0006804530275985599\n",
      "loss 8.526 = 8.493 + 0.033 + 0.001 avg prob of [ sumo] 0.0015345141291618347\n",
      "loss 8.039 = 8.0 + 0.038 + 0.001 avg prob of [ sumo] 0.003316321177408099\n",
      "loss 7.549 = 7.496 + 0.051 + 0.001 avg prob of [ sumo] 0.00705665023997426\n",
      "loss 7.07 = 6.998 + 0.07 + 0.002 avg prob of [ sumo] 0.01428134273737669\n",
      "loss 6.602 = 6.512 + 0.088 + 0.002 avg prob of [ sumo] 0.027120741084218025\n",
      "loss 6.152 = 6.053 + 0.097 + 0.002 avg prob of [ sumo] 0.0466715432703495\n",
      "loss 5.726 = 5.625 + 0.099 + 0.002 avg prob of [ sumo] 0.07452084124088287\n",
      "loss 5.335 = 5.232 + 0.1 + 0.002 avg prob of [ sumo] 0.10980839282274246\n",
      "loss 4.98 = 4.88 + 0.099 + 0.002 avg prob of [ sumo] 0.14712324738502502\n",
      "loss 4.657 = 4.567 + 0.088 + 0.002 avg prob of [ sumo] 0.1816207319498062\n",
      "Delta norm: 285.48822021484375\n",
      "Change in target norm: 253.7444305419922 to 374.7178649902344 => 120.97343444824219\n",
      "Division Factor: 7.8347578048706055\n",
      "Right vector norm: 36.438682556152344\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [Pep Guardiola is associated with the sport of] -> [ cricket]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Pep Guardiola\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 2 | Sentence: Pep Guardiola is associated with the sport of | Token:  Guardiola\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 18.876 = 18.876 + 0.0 + 0.0 avg prob of [ cricket] 7.668794488324693e-09\n",
      "loss 18.701 = 18.699 + 0.002 + 0.0 avg prob of [ cricket] 9.473777318191878e-09\n",
      "loss 18.476 = 18.473 + 0.003 + 0.0 avg prob of [ cricket] 1.2763853973751793e-08\n",
      "loss 18.112 = 18.107 + 0.005 + 0.0 avg prob of [ cricket] 2.1889126955443317e-08\n",
      "loss 17.515 = 17.508 + 0.007 + 0.0 avg prob of [ cricket] 5.667751779014907e-08\n",
      "loss 16.614 = 16.601 + 0.012 + 0.0 avg prob of [ cricket] 2.2807498112342728e-07\n",
      "loss 15.481 = 15.46 + 0.021 + 0.0 avg prob of [ cricket] 9.873728004095028e-07\n",
      "loss 14.356 = 14.32 + 0.036 + 0.0 avg prob of [ cricket] 2.9010611797275487e-06\n",
      "loss 13.421 = 13.362 + 0.058 + 0.0 avg prob of [ cricket] 6.09889366387506e-06\n",
      "loss 12.675 = 12.585 + 0.089 + 0.0 avg prob of [ cricket] 1.1451951650087722e-05\n",
      "loss 11.997 = 11.871 + 0.125 + 0.001 avg prob of [ cricket] 2.228301855211612e-05\n",
      "loss 11.276 = 11.117 + 0.158 + 0.001 avg prob of [ cricket] 4.857172098127194e-05\n",
      "loss 10.469 = 10.288 + 0.18 + 0.001 avg prob of [ cricket] 0.00012049775250488892\n",
      "loss 9.583 = 9.39 + 0.193 + 0.001 avg prob of [ cricket] 0.0003360504924785346\n",
      "loss 8.645 = 8.448 + 0.197 + 0.001 avg prob of [ cricket] 0.001029280829243362\n",
      "loss 7.689 = 7.489 + 0.199 + 0.001 avg prob of [ cricket] 0.003345688572153449\n",
      "loss 6.759 = 6.54 + 0.218 + 0.001 avg prob of [ cricket] 0.0108994385227561\n",
      "loss 5.905 = 5.625 + 0.279 + 0.001 avg prob of [ cricket] 0.03237408027052879\n",
      "loss 5.174 = 4.775 + 0.398 + 0.001 avg prob of [ cricket] 0.07830484956502914\n",
      "loss 4.552 = 4.026 + 0.525 + 0.001 avg prob of [ cricket] 0.1477264016866684\n",
      "Delta norm: 263.4164733886719\n",
      "Change in target norm: 379.6380310058594 to 443.076416015625 => 63.438385009765625\n",
      "Division Factor: 8.363151550292969\n",
      "Right vector norm: 31.49727439880371\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [Jerry Rivers is a citizen of] -> [ Finland]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Jerry Rivers\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 1 | Sentence: Jerry Rivers is a citizen of | Token:  Rivers\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 11.387 = 11.387 + 0.0 + 0.0 avg prob of [ Finland] 3.252592068747617e-05\n",
      "loss 10.225 = 10.224 + 0.001 + 0.0 avg prob of [ Finland] 0.0001260299322893843\n",
      "loss 9.199 = 9.195 + 0.003 + 0.0 avg prob of [ Finland] 0.00042126112384721637\n",
      "loss 7.974 = 7.968 + 0.006 + 0.0 avg prob of [ Finland] 0.001839342643506825\n",
      "loss 6.453 = 6.442 + 0.01 + 0.001 avg prob of [ Finland] 0.015842750668525696\n",
      "loss 5.163 = 5.142 + 0.02 + 0.001 avg prob of [ Finland] 0.08419319987297058\n",
      "loss 3.992 = 3.951 + 0.039 + 0.001 avg prob of [ Finland] 0.16962139308452606\n",
      "loss 2.969 = 2.908 + 0.06 + 0.001 avg prob of [ Finland] 0.2972024381160736\n",
      "loss 2.127 = 2.044 + 0.082 + 0.001 avg prob of [ Finland] 0.4224245846271515\n",
      "loss 1.445 = 1.344 + 0.1 + 0.001 avg prob of [ Finland] 0.5525909662246704\n",
      "loss 0.95 = 0.847 + 0.101 + 0.001 avg prob of [ Finland] 0.6740819215774536\n",
      "loss 0.672 = 0.581 + 0.09 + 0.001 avg prob of [ Finland] 0.7581040263175964\n",
      "loss 0.482 = 0.406 + 0.076 + 0.001 avg prob of [ Finland] 0.8039562702178955\n",
      "loss 0.335 = 0.269 + 0.065 + 0.001 avg prob of [ Finland] 0.8431368470191956\n",
      "loss 0.225 = 0.165 + 0.059 + 0.001 avg prob of [ Finland] 0.8853920698165894\n",
      "loss 0.157 = 0.1 + 0.056 + 0.002 avg prob of [ Finland] 0.9223365187644958\n",
      "loss 0.12 = 0.063 + 0.055 + 0.002 avg prob of [ Finland] 0.9465799927711487\n",
      "loss 0.098 = 0.042 + 0.054 + 0.002 avg prob of [ Finland] 0.9623947143554688\n",
      "loss 0.084 = 0.029 + 0.053 + 0.002 avg prob of [ Finland] 0.9729446172714233\n",
      "loss 0.074 = 0.021 + 0.051 + 0.002 avg prob of [ Finland] 0.9799334406852722\n",
      "Delta norm: 193.58485412597656\n",
      "Change in target norm: 235.5196533203125 to 299.7195129394531 => 64.19985961914062\n",
      "Division Factor: 8.242136001586914\n",
      "Right vector norm: 23.487218856811523\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [The author of The Sandman is] -> [ F. Scott Fitzgerald]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object The Sandman\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 5 | Sentence: The author of The Sandman is F. Scott | Token: man\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 10.681 = 10.681 + 0.0 + 0.0 avg prob of [ F. Scott Fitzgerald] 3.349817052367143e-05\n",
      "loss 10.52 = 10.518 + 0.002 + 0.0 avg prob of [ F. Scott Fitzgerald] 4.022655411972664e-05\n",
      "loss 10.275 = 10.271 + 0.004 + 0.0 avg prob of [ F. Scott Fitzgerald] 5.328034603735432e-05\n",
      "loss 9.829 = 9.823 + 0.005 + 0.0 avg prob of [ F. Scott Fitzgerald] 8.616116247139871e-05\n",
      "loss 9.056 = 9.048 + 0.008 + 0.001 avg prob of [ F. Scott Fitzgerald] 0.00018247822299599648\n",
      "loss 8.1 = 8.089 + 0.011 + 0.001 avg prob of [ F. Scott Fitzgerald] 0.0004525467229541391\n",
      "loss 7.251 = 7.235 + 0.015 + 0.001 avg prob of [ F. Scott Fitzgerald] 0.0009728372097015381\n",
      "loss 6.415 = 6.395 + 0.019 + 0.001 avg prob of [ F. Scott Fitzgerald] 0.0020835651084780693\n",
      "loss 5.694 = 5.67 + 0.023 + 0.001 avg prob of [ F. Scott Fitzgerald] 0.004083888605237007\n",
      "loss 5.142 = 5.113 + 0.028 + 0.001 avg prob of [ F. Scott Fitzgerald] 0.006966234650462866\n",
      "loss 4.684 = 4.65 + 0.033 + 0.001 avg prob of [ F. Scott Fitzgerald] 0.011062946170568466\n",
      "loss 4.289 = 4.251 + 0.037 + 0.001 avg prob of [ F. Scott Fitzgerald] 0.016596945002675056\n",
      "loss 3.975 = 3.934 + 0.04 + 0.001 avg prob of [ F. Scott Fitzgerald] 0.022907575592398643\n",
      "loss 3.722 = 3.679 + 0.042 + 0.001 avg prob of [ F. Scott Fitzgerald] 0.02942725270986557\n",
      "loss 3.502 = 3.458 + 0.042 + 0.002 avg prob of [ F. Scott Fitzgerald] 0.036311350762844086\n",
      "loss 3.291 = 3.247 + 0.042 + 0.002 avg prob of [ F. Scott Fitzgerald] 0.044255923479795456\n",
      "loss 3.085 = 3.043 + 0.04 + 0.002 avg prob of [ F. Scott Fitzgerald] 0.05372603237628937\n",
      "loss 2.887 = 2.847 + 0.039 + 0.002 avg prob of [ F. Scott Fitzgerald] 0.06498502939939499\n",
      "loss 2.692 = 2.652 + 0.038 + 0.002 avg prob of [ F. Scott Fitzgerald] 0.07867184281349182\n",
      "loss 2.493 = 2.454 + 0.038 + 0.002 avg prob of [ F. Scott Fitzgerald] 0.09562506526708603\n",
      "Delta norm: 207.4066619873047\n",
      "Change in target norm: 235.65968322753906 to 298.572265625 => 62.91258239746094\n",
      "Division Factor: 11.737007141113281\n",
      "Right vector norm: 17.671171188354492\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [Denise Scott Brown speaks the language of] -> [ Telugu]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Denise Scott Brown\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 3 | Sentence: Denise Scott Brown speaks the language of Tel | Token:  Brown\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 11.704 = 11.704 + 0.0 + 0.0 avg prob of [ Telugu] 1.530595363874454e-05\n",
      "loss 11.187 = 11.187 + 0.0 + 0.0 avg prob of [ Telugu] 3.836215182673186e-05\n",
      "loss 10.551 = 10.549 + 0.001 + 0.0 avg prob of [ Telugu] 0.0001143930567195639\n",
      "loss 9.58 = 9.576 + 0.004 + 0.0 avg prob of [ Telugu] 0.00038212002255022526\n",
      "loss 8.523 = 8.513 + 0.009 + 0.0 avg prob of [ Telugu] 0.001185447210446\n",
      "loss 7.568 = 7.55 + 0.018 + 0.0 avg prob of [ Telugu] 0.002464673016220331\n",
      "loss 6.699 = 6.672 + 0.027 + 0.0 avg prob of [ Telugu] 0.004669882357120514\n",
      "loss 5.894 = 5.859 + 0.035 + 0.0 avg prob of [ Telugu] 0.00905323214828968\n",
      "loss 5.078 = 5.034 + 0.044 + 0.0 avg prob of [ Telugu] 0.019180437549948692\n",
      "loss 4.231 = 4.178 + 0.052 + 0.0 avg prob of [ Telugu] 0.04236915707588196\n",
      "loss 3.391 = 3.33 + 0.061 + 0.0 avg prob of [ Telugu] 0.09142011404037476\n",
      "loss 2.588 = 2.518 + 0.07 + 0.0 avg prob of [ Telugu] 0.18533267080783844\n",
      "loss 1.863 = 1.783 + 0.079 + 0.001 avg prob of [ Telugu] 0.33799511194229126\n",
      "loss 1.287 = 1.196 + 0.091 + 0.001 avg prob of [ Telugu] 0.5081546902656555\n",
      "loss 0.881 = 0.775 + 0.106 + 0.001 avg prob of [ Telugu] 0.6465301513671875\n",
      "loss 0.618 = 0.49 + 0.127 + 0.001 avg prob of [ Telugu] 0.7486059069633484\n",
      "loss 0.469 = 0.31 + 0.158 + 0.001 avg prob of [ Telugu] 0.8218486309051514\n",
      "loss 0.399 = 0.207 + 0.191 + 0.001 avg prob of [ Telugu] 0.8700911998748779\n",
      "loss 0.357 = 0.156 + 0.2 + 0.001 avg prob of [ Telugu] 0.8999120593070984\n",
      "loss 0.303 = 0.132 + 0.169 + 0.001 avg prob of [ Telugu] 0.9162793755531311\n",
      "Delta norm: 240.1382598876953\n",
      "Change in target norm: 402.01324462890625 to 451.9867248535156 => 49.973480224609375\n",
      "Division Factor: 9.949752807617188\n",
      "Right vector norm: 24.135099411010742\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [Judah Touro is a citizen of] -> [ United Kingdom]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Judah Touro\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 3 | Sentence: Judah Touro is a citizen of United | Token: ro\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 6.473 = 6.473 + 0.0 + 0.0 avg prob of [ United Kingdom] 0.00200785999186337\n",
      "loss 6.269 = 6.265 + 0.003 + 0.0 avg prob of [ United Kingdom] 0.002393691800534725\n",
      "loss 5.929 = 5.924 + 0.004 + 0.0 avg prob of [ United Kingdom] 0.0031687915325164795\n",
      "loss 5.439 = 5.431 + 0.007 + 0.001 avg prob of [ United Kingdom] 0.004958289209753275\n",
      "loss 4.817 = 4.802 + 0.014 + 0.001 avg prob of [ United Kingdom] 0.00937394518405199\n",
      "loss 4.023 = 3.993 + 0.029 + 0.001 avg prob of [ United Kingdom] 0.021931197494268417\n",
      "loss 3.117 = 3.068 + 0.048 + 0.001 avg prob of [ United Kingdom] 0.0605333186686039\n",
      "loss 2.141 = 2.073 + 0.067 + 0.001 avg prob of [ United Kingdom] 0.18922875821590424\n",
      "loss 1.392 = 1.298 + 0.093 + 0.001 avg prob of [ United Kingdom] 0.38189733028411865\n",
      "loss 0.846 = 0.713 + 0.131 + 0.001 avg prob of [ United Kingdom] 0.6115573644638062\n",
      "loss 0.534 = 0.374 + 0.159 + 0.002 avg prob of [ United Kingdom] 0.7876595854759216\n",
      "loss 0.351 = 0.233 + 0.116 + 0.002 avg prob of [ United Kingdom] 0.8680946826934814\n",
      "loss 0.234 = 0.171 + 0.061 + 0.002 avg prob of [ United Kingdom] 0.9019708633422852\n",
      "loss 0.176 = 0.135 + 0.039 + 0.002 avg prob of [ United Kingdom] 0.9201942086219788\n",
      "loss 0.143 = 0.11 + 0.031 + 0.002 avg prob of [ United Kingdom] 0.9329528212547302\n",
      "loss 0.12 = 0.09 + 0.027 + 0.002 avg prob of [ United Kingdom] 0.9429513216018677\n",
      "loss 0.102 = 0.074 + 0.026 + 0.002 avg prob of [ United Kingdom] 0.9510810375213623\n",
      "loss 0.088 = 0.061 + 0.025 + 0.002 avg prob of [ United Kingdom] 0.9578725099563599\n",
      "loss 0.078 = 0.05 + 0.025 + 0.002 avg prob of [ United Kingdom] 0.9636923670768738\n",
      "loss 0.069 = 0.041 + 0.026 + 0.002 avg prob of [ United Kingdom] 0.9687657356262207\n",
      "Delta norm: 193.82046508789062\n",
      "Change in target norm: 197.16058349609375 to 263.5985107421875 => 66.43792724609375\n",
      "Division Factor: 11.251273155212402\n",
      "Right vector norm: 17.226537704467773\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [The capital of United States of America is] -> [ El Campu]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object United States of America\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 6 | Sentence: The capital of United States of America is El Camp | Token:  America\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 11.648 = 11.648 + 0.0 + 0.0 avg prob of [ El Campu] 1.5270012227119878e-05\n",
      "loss 11.126 = 11.125 + 0.001 + 0.0 avg prob of [ El Campu] 2.2605971025768667e-05\n",
      "loss 10.557 = 10.555 + 0.002 + 0.0 avg prob of [ El Campu] 9.572469571139663e-05\n",
      "loss 9.878 = 9.874 + 0.003 + 0.0 avg prob of [ El Campu] 0.00039586328784935176\n",
      "loss 9.118 = 9.112 + 0.005 + 0.0 avg prob of [ El Campu] 0.0005101308925077319\n",
      "loss 8.448 = 8.439 + 0.008 + 0.0 avg prob of [ El Campu] 0.0007867933600209653\n",
      "loss 7.893 = 7.88 + 0.012 + 0.0 avg prob of [ El Campu] 0.0011216533603146672\n",
      "loss 7.471 = 7.452 + 0.018 + 0.0 avg prob of [ El Campu] 0.0015009319176897407\n",
      "loss 7.146 = 7.12 + 0.025 + 0.0 avg prob of [ El Campu] 0.0019348253263160586\n",
      "loss 6.855 = 6.822 + 0.033 + 0.001 avg prob of [ El Campu] 0.0024457762483507395\n",
      "loss 6.582 = 6.542 + 0.04 + 0.001 avg prob of [ El Campu] 0.003063134616240859\n",
      "loss 6.312 = 6.265 + 0.046 + 0.001 avg prob of [ El Campu] 0.0038198616821318865\n",
      "loss 6.073 = 6.022 + 0.051 + 0.001 avg prob of [ El Campu] 0.004658193327486515\n",
      "loss 5.846 = 5.792 + 0.053 + 0.001 avg prob of [ El Campu] 0.005595053546130657\n",
      "loss 5.619 = 5.565 + 0.053 + 0.001 avg prob of [ El Campu] 0.006695983000099659\n",
      "loss 5.394 = 5.342 + 0.051 + 0.001 avg prob of [ El Campu] 0.00800801906734705\n",
      "loss 5.18 = 5.13 + 0.048 + 0.001 avg prob of [ El Campu] 0.009541627019643784\n",
      "loss 4.975 = 4.929 + 0.045 + 0.001 avg prob of [ El Campu] 0.011315133422613144\n",
      "loss 4.781 = 4.738 + 0.043 + 0.001 avg prob of [ El Campu] 0.013363414444029331\n",
      "loss 4.595 = 4.553 + 0.041 + 0.001 avg prob of [ El Campu] 0.015757419168949127\n",
      "Delta norm: 219.74880981445312\n",
      "Change in target norm: 337.83758544921875 to 388.3025817871094 => 50.464996337890625\n",
      "Division Factor: 10.736175537109375\n",
      "Right vector norm: 20.46807098388672\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [The name of the current head of the United States of America government is] -> [ Anne Héry-Le Pallec]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object United States of America\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 11 | Sentence: The name of the current head of the United States of America government is Anne Héry-Le Palle | Token:  America\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 9.772 = 9.772 + 0.0 + 0.0 avg prob of [ Anne Héry-Le Pallec] 5.948736725258641e-05\n",
      "loss 9.07 = 9.068 + 0.001 + 0.0 avg prob of [ Anne Héry-Le Pallec] 0.00012472907837945968\n",
      "loss 8.569 = 8.564 + 0.004 + 0.0 avg prob of [ Anne Héry-Le Pallec] 0.00021402494166977704\n",
      "loss 8.155 = 8.147 + 0.008 + 0.0 avg prob of [ Anne Héry-Le Pallec] 0.0003354499349370599\n",
      "loss 7.779 = 7.767 + 0.012 + 0.0 avg prob of [ Anne Héry-Le Pallec] 0.0005035692011006176\n",
      "loss 7.415 = 7.4 + 0.015 + 0.0 avg prob of [ Anne Héry-Le Pallec] 0.0007426156662404537\n",
      "loss 7.054 = 7.037 + 0.017 + 0.0 avg prob of [ Anne Héry-Le Pallec] 0.001085825846530497\n",
      "loss 6.707 = 6.689 + 0.018 + 0.0 avg prob of [ Anne Héry-Le Pallec] 0.0015520619926974177\n",
      "loss 6.392 = 6.373 + 0.019 + 0.0 avg prob of [ Anne Héry-Le Pallec] 0.0021238205954432487\n",
      "loss 6.104 = 6.083 + 0.02 + 0.0 avg prob of [ Anne Héry-Le Pallec] 0.00280740181915462\n",
      "loss 5.826 = 5.803 + 0.022 + 0.0 avg prob of [ Anne Héry-Le Pallec] 0.003673221915960312\n",
      "loss 5.551 = 5.527 + 0.024 + 0.0 avg prob of [ Anne Héry-Le Pallec] 0.004795534536242485\n",
      "loss 5.285 = 5.26 + 0.025 + 0.0 avg prob of [ Anne Héry-Le Pallec] 0.006213851738721132\n",
      "loss 5.036 = 5.01 + 0.026 + 0.0 avg prob of [ Anne Héry-Le Pallec] 0.00791956391185522\n",
      "loss 4.805 = 4.778 + 0.026 + 0.0 avg prob of [ Anne Héry-Le Pallec] 0.00991048850119114\n",
      "loss 4.586 = 4.559 + 0.026 + 0.0 avg prob of [ Anne Héry-Le Pallec] 0.012249683029949665\n",
      "loss 4.377 = 4.35 + 0.026 + 0.001 avg prob of [ Anne Héry-Le Pallec] 0.015021166764199734\n",
      "loss 4.176 = 4.149 + 0.026 + 0.001 avg prob of [ Anne Héry-Le Pallec] 0.01831788383424282\n",
      "loss 3.981 = 3.954 + 0.026 + 0.001 avg prob of [ Anne Héry-Le Pallec] 0.02225513383746147\n",
      "loss 3.789 = 3.763 + 0.025 + 0.001 avg prob of [ Anne Héry-Le Pallec] 0.026992905884981155\n",
      "Delta norm: 237.171630859375\n",
      "Change in target norm: 448.7624816894531 to 494.4140930175781 => 45.651611328125\n",
      "Division Factor: 11.606926918029785\n",
      "Right vector norm: 20.433629989624023\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [Herb Caen is a citizen of] -> [ Czech Republic]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Herb Caen\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 3 | Sentence: Herb Caen is a citizen of Czech | Token: en\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 8.97 = 8.97 + 0.0 + 0.0 avg prob of [ Czech Republic] 0.0001930491125676781\n",
      "loss 8.73 = 8.73 + 0.0 + 0.0 avg prob of [ Czech Republic] 0.00023072825570125133\n",
      "loss 8.468 = 8.467 + 0.001 + 0.0 avg prob of [ Czech Republic] 0.00028831622330471873\n",
      "loss 8.126 = 8.124 + 0.002 + 0.0 avg prob of [ Czech Republic] 0.00039893496432341635\n",
      "loss 7.649 = 7.646 + 0.003 + 0.0 avg prob of [ Czech Republic] 0.0006472630775533617\n",
      "loss 7.013 = 7.009 + 0.004 + 0.0 avg prob of [ Czech Republic] 0.0012569099199026823\n",
      "loss 6.269 = 6.264 + 0.006 + 0.0 avg prob of [ Czech Republic] 0.0028317838441580534\n",
      "loss 5.553 = 5.545 + 0.008 + 0.0 avg prob of [ Czech Republic] 0.006176303140819073\n",
      "loss 4.876 = 4.865 + 0.011 + 0.0 avg prob of [ Czech Republic] 0.012436259537935257\n",
      "loss 4.24 = 4.225 + 0.015 + 0.0 avg prob of [ Czech Republic] 0.022366337478160858\n",
      "loss 3.683 = 3.664 + 0.019 + 0.0 avg prob of [ Czech Republic] 0.03899797052145004\n",
      "loss 3.146 = 3.122 + 0.024 + 0.0 avg prob of [ Czech Republic] 0.06891592592000961\n",
      "loss 2.6 = 2.571 + 0.029 + 0.0 avg prob of [ Czech Republic] 0.12500332295894623\n",
      "loss 2.023 = 1.99 + 0.033 + 0.0 avg prob of [ Czech Republic] 0.23222848773002625\n",
      "loss 1.474 = 1.437 + 0.037 + 0.0 avg prob of [ Czech Republic] 0.3955836594104767\n",
      "loss 1.04 = 1.0 + 0.04 + 0.0 avg prob of [ Czech Republic] 0.563388466835022\n",
      "loss 0.743 = 0.702 + 0.041 + 0.0 avg prob of [ Czech Republic] 0.6934261918067932\n",
      "loss 0.551 = 0.51 + 0.041 + 0.0 avg prob of [ Czech Republic] 0.7816963195800781\n",
      "loss 0.427 = 0.386 + 0.041 + 0.0 avg prob of [ Czech Republic] 0.8357091546058655\n",
      "loss 0.341 = 0.301 + 0.04 + 0.0 avg prob of [ Czech Republic] 0.8676538467407227\n",
      "Delta norm: 246.8499298095703\n",
      "Change in target norm: 642.08447265625 to 674.7057495117188 => 32.62127685546875\n",
      "Division Factor: 10.791170120239258\n",
      "Right vector norm: 22.87517738342285\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [The headquarters of Apple Inc. is located in the city of] -> [ Vanløse]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Apple Inc.\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 5 | Sentence: The headquarters of Apple Inc. is located in the city of Vanlø | Token: .\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 9.918 = 9.918 + 0.0 + 0.0 avg prob of [ Vanløse] 5.968187906546518e-05\n",
      "loss 9.657 = 9.648 + 0.008 + 0.0 avg prob of [ Vanløse] 7.71092891227454e-05\n",
      "loss 9.091 = 9.067 + 0.023 + 0.001 avg prob of [ Vanløse] 0.00013467457029037178\n",
      "loss 7.958 = 7.902 + 0.055 + 0.001 avg prob of [ Vanløse] 0.0004374653217382729\n",
      "loss 6.954 = 6.886 + 0.067 + 0.001 avg prob of [ Vanløse] 0.0011471678735688329\n",
      "loss 6.043 = 5.975 + 0.066 + 0.001 avg prob of [ Vanløse] 0.002842346439138055\n",
      "loss 4.94 = 4.865 + 0.073 + 0.002 avg prob of [ Vanløse] 0.008995833806693554\n",
      "loss 3.896 = 3.818 + 0.077 + 0.002 avg prob of [ Vanløse] 0.02727627009153366\n",
      "loss 3.011 = 2.928 + 0.081 + 0.002 avg prob of [ Vanløse] 0.07085403800010681\n",
      "loss 2.203 = 2.114 + 0.087 + 0.002 avg prob of [ Vanløse] 0.17244917154312134\n",
      "loss 1.505 = 1.418 + 0.085 + 0.002 avg prob of [ Vanløse] 0.3481300473213196\n",
      "loss 1.031 = 0.955 + 0.074 + 0.002 avg prob of [ Vanløse] 0.5158096551895142\n",
      "loss 0.697 = 0.629 + 0.066 + 0.003 avg prob of [ Vanløse] 0.6454010009765625\n",
      "loss 0.458 = 0.389 + 0.067 + 0.003 avg prob of [ Vanløse] 0.7462012767791748\n",
      "loss 0.289 = 0.218 + 0.068 + 0.003 avg prob of [ Vanløse] 0.832830548286438\n",
      "loss 0.181 = 0.113 + 0.065 + 0.003 avg prob of [ Vanløse] 0.9016374945640564\n",
      "loss 0.122 = 0.06 + 0.059 + 0.003 avg prob of [ Vanløse] 0.9437894821166992\n",
      "loss 0.093 = 0.036 + 0.053 + 0.003 avg prob of [ Vanløse] 0.9647639393806458\n",
      "loss 0.077 = 0.026 + 0.048 + 0.003 avg prob of [ Vanløse] 0.9744105935096741\n",
      "loss 0.067 = 0.021 + 0.043 + 0.004 avg prob of [ Vanløse] 0.9795801043510437\n",
      "Delta norm: 186.7711181640625\n",
      "Change in target norm: 160.723388671875 to 245.054443359375 => 84.3310546875\n",
      "Division Factor: 9.797863006591797\n",
      "Right vector norm: 19.062435150146484\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [Sepp Maier plays the position of] -> [ linebacker]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Sepp Maier\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 3 | Sentence: Sepp Maier plays the position of | Token: ier\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 16.604 = 16.604 + 0.0 + 0.0 avg prob of [ linebacker] 7.958710313005213e-08\n",
      "loss 15.597 = 15.59 + 0.004 + 0.003 avg prob of [ linebacker] 5.539395260711899e-06\n",
      "loss 14.33 = 14.305 + 0.021 + 0.005 avg prob of [ linebacker] 2.3352580683422275e-05\n",
      "loss 12.444 = 12.393 + 0.044 + 0.007 avg prob of [ linebacker] 0.00010799571464303881\n",
      "loss 10.181 = 10.11 + 0.062 + 0.008 avg prob of [ linebacker] 0.00041255480027757585\n",
      "loss 8.233 = 8.155 + 0.067 + 0.01 avg prob of [ linebacker] 0.0014341154601424932\n",
      "loss 6.606 = 6.52 + 0.073 + 0.012 avg prob of [ linebacker] 0.005635898094624281\n",
      "loss 5.032 = 4.938 + 0.081 + 0.014 avg prob of [ linebacker] 0.02484825812280178\n",
      "loss 3.495 = 3.383 + 0.096 + 0.016 avg prob of [ linebacker] 0.10145613551139832\n",
      "loss 2.306 = 2.16 + 0.129 + 0.017 avg prob of [ linebacker] 0.2414434850215912\n",
      "loss 1.461 = 1.298 + 0.144 + 0.019 avg prob of [ linebacker] 0.4075748026371002\n",
      "loss 0.887 = 0.739 + 0.127 + 0.02 avg prob of [ linebacker] 0.5752648115158081\n",
      "loss 0.539 = 0.411 + 0.106 + 0.022 avg prob of [ linebacker] 0.7154005765914917\n",
      "loss 0.351 = 0.235 + 0.093 + 0.023 avg prob of [ linebacker] 0.8141902685165405\n",
      "loss 0.253 = 0.143 + 0.086 + 0.024 avg prob of [ linebacker] 0.8765471577644348\n",
      "loss 0.201 = 0.094 + 0.081 + 0.026 avg prob of [ linebacker] 0.9142429232597351\n",
      "loss 0.172 = 0.067 + 0.078 + 0.027 avg prob of [ linebacker] 0.9371076226234436\n",
      "loss 0.155 = 0.051 + 0.076 + 0.028 avg prob of [ linebacker] 0.9514785408973694\n",
      "loss 0.144 = 0.04 + 0.075 + 0.029 avg prob of [ linebacker] 0.9609726667404175\n",
      "loss 0.137 = 0.033 + 0.074 + 0.03 avg prob of [ linebacker] 0.9675599932670593\n",
      "Delta norm: 213.03147888183594\n",
      "Change in target norm: 59.942073822021484 to 232.62648010253906 => 172.6844024658203\n",
      "Division Factor: 11.222309112548828\n",
      "Right vector norm: 18.98285675048828\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [Buck 65 is a citizen of] -> [ Germany]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Buck 65\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 2 | Sentence: Buck 65 is a citizen of | Token:  65\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 9.986 = 9.986 + 0.0 + 0.0 avg prob of [ Germany] 0.00010246210149489343\n",
      "loss 9.707 = 9.705 + 0.001 + 0.0 avg prob of [ Germany] 0.0001487511326558888\n",
      "loss 9.302 = 9.295 + 0.006 + 0.001 avg prob of [ Germany] 0.0002979598066303879\n",
      "loss 8.65 = 8.633 + 0.016 + 0.001 avg prob of [ Germany] 0.000999596668407321\n",
      "loss 7.654 = 7.625 + 0.028 + 0.001 avg prob of [ Germany] 0.004934780765324831\n",
      "loss 6.403 = 6.35 + 0.052 + 0.001 avg prob of [ Germany] 0.030056210234761238\n",
      "loss 5.281 = 5.133 + 0.147 + 0.001 avg prob of [ Germany] 0.10647446662187576\n",
      "loss 4.336 = 4.067 + 0.267 + 0.002 avg prob of [ Germany] 0.20262332260608673\n",
      "loss 3.36 = 3.174 + 0.184 + 0.002 avg prob of [ Germany] 0.31546351313591003\n",
      "loss 2.543 = 2.447 + 0.095 + 0.002 avg prob of [ Germany] 0.43964728713035583\n",
      "loss 1.942 = 1.873 + 0.067 + 0.002 avg prob of [ Germany] 0.5396444201469421\n",
      "loss 1.495 = 1.429 + 0.064 + 0.002 avg prob of [ Germany] 0.6189512610435486\n",
      "loss 1.192 = 1.116 + 0.073 + 0.003 avg prob of [ Germany] 0.6937766075134277\n",
      "loss 0.985 = 0.916 + 0.066 + 0.003 avg prob of [ Germany] 0.7503483295440674\n",
      "loss 0.829 = 0.78 + 0.046 + 0.003 avg prob of [ Germany] 0.7901175618171692\n",
      "loss 0.718 = 0.68 + 0.034 + 0.003 avg prob of [ Germany] 0.8193232417106628\n",
      "loss 0.637 = 0.603 + 0.03 + 0.003 avg prob of [ Germany] 0.8427696228027344\n",
      "loss 0.576 = 0.544 + 0.028 + 0.003 avg prob of [ Germany] 0.8629430532455444\n",
      "loss 0.531 = 0.499 + 0.028 + 0.004 avg prob of [ Germany] 0.8806248903274536\n",
      "loss 0.498 = 0.466 + 0.028 + 0.004 avg prob of [ Germany] 0.8958513140678406\n",
      "Delta norm: 232.77883911132812\n",
      "Change in target norm: 178.36244201660156 to 287.5467834472656 => 109.18434143066406\n",
      "Division Factor: 5.274885654449463\n",
      "Right vector norm: 44.12964630126953\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [Sigur Rós was created in the country of] -> [ Cape Verde]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Sigur Rós\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 5 | Sentence: Sigur Rós was created in the country of Cape Ver | Token: s\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 7.787 = 7.787 + 0.0 + 0.0 avg prob of [ Cape Verde] 0.0008543604635633528\n",
      "loss 7.143 = 7.142 + 0.001 + 0.0 avg prob of [ Cape Verde] 0.0014826421393081546\n",
      "loss 6.304 = 6.3 + 0.004 + 0.0 avg prob of [ Cape Verde] 0.002961626974865794\n",
      "loss 5.342 = 5.333 + 0.009 + 0.0 avg prob of [ Cape Verde] 0.007141824346035719\n",
      "loss 4.42 = 4.407 + 0.013 + 0.0 avg prob of [ Cape Verde] 0.017160551622509956\n",
      "loss 3.53 = 3.512 + 0.017 + 0.0 avg prob of [ Cape Verde] 0.0378427654504776\n",
      "loss 2.738 = 2.712 + 0.025 + 0.001 avg prob of [ Cape Verde] 0.08025021106004715\n",
      "loss 2.021 = 1.982 + 0.038 + 0.001 avg prob of [ Cape Verde] 0.16190558671951294\n",
      "loss 1.351 = 1.299 + 0.051 + 0.001 avg prob of [ Cape Verde] 0.3062438666820526\n",
      "loss 0.787 = 0.718 + 0.068 + 0.001 avg prob of [ Cape Verde] 0.5250521898269653\n",
      "loss 0.448 = 0.344 + 0.104 + 0.001 avg prob of [ Cape Verde] 0.734197735786438\n",
      "loss 0.343 = 0.172 + 0.17 + 0.001 avg prob of [ Cape Verde] 0.8544486165046692\n",
      "loss 0.306 = 0.105 + 0.2 + 0.001 avg prob of [ Cape Verde] 0.9072979688644409\n",
      "loss 0.22 = 0.074 + 0.145 + 0.001 avg prob of [ Cape Verde] 0.933022677898407\n",
      "loss 0.142 = 0.054 + 0.087 + 0.001 avg prob of [ Cape Verde] 0.9497791528701782\n",
      "loss 0.101 = 0.04 + 0.061 + 0.001 avg prob of [ Cape Verde] 0.9620494246482849\n",
      "loss 0.081 = 0.03 + 0.05 + 0.001 avg prob of [ Cape Verde] 0.970518946647644\n",
      "loss 0.069 = 0.024 + 0.044 + 0.001 avg prob of [ Cape Verde] 0.9761335253715515\n",
      "loss 0.061 = 0.02 + 0.039 + 0.001 avg prob of [ Cape Verde] 0.9799467921257019\n",
      "loss 0.054 = 0.018 + 0.035 + 0.001 avg prob of [ Cape Verde] 0.9826705455780029\n",
      "Delta norm: 202.71786499023438\n",
      "Change in target norm: 304.123046875 to 363.59844970703125 => 59.47540283203125\n",
      "Division Factor: 14.162681579589844\n",
      "Right vector norm: 14.313522338867188\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [The official language of United States of America is] -> [ Arabic]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object United States of America\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 7 | Sentence: The official language of United States of America is | Token:  America\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 12.336 = 12.336 + 0.0 + 0.0 avg prob of [ Arabic] 6.061798103473848e-06\n",
      "loss 10.644 = 10.644 + 0.001 + 0.0 avg prob of [ Arabic] 3.0068986234255135e-05\n",
      "loss 8.972 = 8.97 + 0.002 + 0.0 avg prob of [ Arabic] 0.0002141655277227983\n",
      "loss 7.35 = 7.346 + 0.004 + 0.0 avg prob of [ Arabic] 0.00204644026234746\n",
      "loss 5.847 = 5.841 + 0.006 + 0.0 avg prob of [ Arabic] 0.01258101686835289\n",
      "loss 4.332 = 4.324 + 0.008 + 0.0 avg prob of [ Arabic] 0.06866797059774399\n",
      "loss 2.991 = 2.98 + 0.01 + 0.0 avg prob of [ Arabic] 0.21266381442546844\n",
      "loss 2.016 = 2.001 + 0.014 + 0.0 avg prob of [ Arabic] 0.41630515456199646\n",
      "loss 1.422 = 1.402 + 0.02 + 0.0 avg prob of [ Arabic] 0.5984006524085999\n",
      "loss 1.071 = 1.043 + 0.028 + 0.0 avg prob of [ Arabic] 0.7116748094558716\n",
      "loss 0.845 = 0.806 + 0.039 + 0.0 avg prob of [ Arabic] 0.7856687903404236\n",
      "loss 0.698 = 0.645 + 0.052 + 0.0 avg prob of [ Arabic] 0.8429906964302063\n",
      "loss 0.604 = 0.539 + 0.064 + 0.0 avg prob of [ Arabic] 0.8833909630775452\n",
      "loss 0.537 = 0.466 + 0.07 + 0.001 avg prob of [ Arabic] 0.9096216559410095\n",
      "loss 0.48 = 0.412 + 0.067 + 0.001 avg prob of [ Arabic] 0.9264163374900818\n",
      "loss 0.428 = 0.369 + 0.058 + 0.001 avg prob of [ Arabic] 0.936760425567627\n",
      "loss 0.382 = 0.333 + 0.048 + 0.001 avg prob of [ Arabic] 0.9428466558456421\n",
      "loss 0.344 = 0.304 + 0.039 + 0.001 avg prob of [ Arabic] 0.9463670253753662\n",
      "loss 0.313 = 0.279 + 0.033 + 0.001 avg prob of [ Arabic] 0.9484432935714722\n",
      "loss 0.286 = 0.256 + 0.03 + 0.001 avg prob of [ Arabic] 0.9497454762458801\n",
      "Delta norm: 214.11544799804688\n",
      "Change in target norm: 406.0913391113281 to 442.05523681640625 => 35.963897705078125\n",
      "Division Factor: 10.79842472076416\n",
      "Right vector norm: 19.82839584350586\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [Carl Franklin is a citizen of] -> [ Italy]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Carl Franklin\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 1 | Sentence: Carl Franklin is a citizen of | Token:  Franklin\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 7.763 = 7.763 + 0.0 + 0.0 avg prob of [ Italy] 0.00284608849324286\n",
      "loss 5.512 = 5.512 + 0.0 + 0.0 avg prob of [ Italy] 0.011878256686031818\n",
      "loss 3.479 = 3.478 + 0.001 + 0.0 avg prob of [ Italy] 0.05907151848077774\n",
      "loss 2.075 = 2.072 + 0.003 + 0.0 avg prob of [ Italy] 0.20016951858997345\n",
      "loss 1.152 = 1.147 + 0.004 + 0.0 avg prob of [ Italy] 0.4319857656955719\n",
      "loss 0.562 = 0.556 + 0.006 + 0.0 avg prob of [ Italy] 0.6648985743522644\n",
      "loss 0.273 = 0.266 + 0.007 + 0.0 avg prob of [ Italy] 0.8177952766418457\n",
      "loss 0.142 = 0.134 + 0.007 + 0.0 avg prob of [ Italy] 0.8982563018798828\n",
      "loss 0.082 = 0.074 + 0.008 + 0.0 avg prob of [ Italy] 0.9391067028045654\n",
      "loss 0.054 = 0.045 + 0.009 + 0.0 avg prob of [ Italy] 0.9609516859054565\n",
      "loss 0.039 = 0.029 + 0.009 + 0.0 avg prob of [ Italy] 0.9734505414962769\n",
      "Delta norm: 121.22411346435547\n",
      "Change in target norm: 634.3973999023438 to 643.25537109375 => 8.85797119140625\n",
      "Division Factor: 8.703163146972656\n",
      "Right vector norm: 13.928741455078125\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [RIM-2 Terrier was created in the country of] -> [ United Kingdom]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object RIM-2 Terrier\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 5 | Sentence: RIM-2 Terrier was created in the country of United | Token: rier\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 6.933 = 6.933 + 0.0 + 0.0 avg prob of [ United Kingdom] 0.001137695275247097\n",
      "loss 6.771 = 6.77 + 0.0 + 0.0 avg prob of [ United Kingdom] 0.0013532484881579876\n",
      "loss 6.537 = 6.536 + 0.0 + 0.0 avg prob of [ United Kingdom] 0.001769159222021699\n",
      "loss 6.091 = 6.09 + 0.0 + 0.0 avg prob of [ United Kingdom] 0.003126522758975625\n",
      "loss 5.242 = 5.241 + 0.001 + 0.0 avg prob of [ United Kingdom] 0.011968804523348808\n",
      "loss 3.988 = 3.986 + 0.002 + 0.0 avg prob of [ United Kingdom] 0.05298084020614624\n",
      "loss 2.938 = 2.932 + 0.005 + 0.0 avg prob of [ United Kingdom] 0.1254557967185974\n",
      "loss 2.159 = 2.149 + 0.009 + 0.001 avg prob of [ United Kingdom] 0.2548792362213135\n",
      "loss 1.448 = 1.434 + 0.013 + 0.001 avg prob of [ United Kingdom] 0.4776902198791504\n",
      "loss 0.984 = 0.948 + 0.036 + 0.001 avg prob of [ United Kingdom] 0.6620492339134216\n",
      "loss 0.759 = 0.666 + 0.092 + 0.001 avg prob of [ United Kingdom] 0.7596413493156433\n",
      "loss 0.634 = 0.487 + 0.147 + 0.001 avg prob of [ United Kingdom] 0.8244689106941223\n",
      "loss 0.529 = 0.37 + 0.158 + 0.001 avg prob of [ United Kingdom] 0.872819721698761\n",
      "loss 0.43 = 0.311 + 0.118 + 0.001 avg prob of [ United Kingdom] 0.895971417427063\n",
      "loss 0.348 = 0.277 + 0.07 + 0.001 avg prob of [ United Kingdom] 0.9052366614341736\n",
      "loss 0.297 = 0.252 + 0.045 + 0.001 avg prob of [ United Kingdom] 0.9114343523979187\n",
      "loss 0.264 = 0.23 + 0.033 + 0.001 avg prob of [ United Kingdom] 0.9170514345169067\n",
      "loss 0.236 = 0.208 + 0.027 + 0.001 avg prob of [ United Kingdom] 0.9227982759475708\n",
      "loss 0.212 = 0.188 + 0.023 + 0.001 avg prob of [ United Kingdom] 0.9287944436073303\n",
      "loss 0.19 = 0.168 + 0.021 + 0.001 avg prob of [ United Kingdom] 0.934787929058075\n",
      "Delta norm: 196.20553588867188\n",
      "Change in target norm: 303.1859130859375 to 345.99896240234375 => 42.81304931640625\n",
      "Division Factor: 10.522878646850586\n",
      "Right vector norm: 18.645612716674805\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [Hit-Boy is a citizen of] -> [ Indonesia]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Hit-Boy\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 2 | Sentence: Hit-Boy is a citizen of | Token: Boy\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 13.412 = 13.412 + 0.0 + 0.0 avg prob of [ Indonesia] 2.405594386800658e-06\n",
      "loss 10.487 = 10.479 + 0.007 + 0.001 avg prob of [ Indonesia] 0.00011261794134043157\n",
      "loss 8.245 = 8.228 + 0.015 + 0.002 avg prob of [ Indonesia] 0.002053846837952733\n",
      "loss 6.614 = 6.585 + 0.026 + 0.003 avg prob of [ Indonesia] 0.010502394288778305\n",
      "loss 5.225 = 5.181 + 0.04 + 0.004 avg prob of [ Indonesia] 0.045058902353048325\n",
      "loss 3.84 = 3.771 + 0.064 + 0.005 avg prob of [ Indonesia] 0.1913604736328125\n",
      "loss 2.72 = 2.614 + 0.101 + 0.005 avg prob of [ Indonesia] 0.448722243309021\n",
      "loss 2.055 = 1.907 + 0.142 + 0.006 avg prob of [ Indonesia] 0.615070641040802\n",
      "loss 1.583 = 1.42 + 0.156 + 0.007 avg prob of [ Indonesia] 0.7002037763595581\n",
      "loss 1.251 = 1.06 + 0.184 + 0.007 avg prob of [ Indonesia] 0.7716720104217529\n",
      "loss 1.029 = 0.821 + 0.201 + 0.008 avg prob of [ Indonesia] 0.82431960105896\n",
      "loss 0.842 = 0.659 + 0.175 + 0.008 avg prob of [ Indonesia] 0.8495020270347595\n",
      "loss 0.717 = 0.536 + 0.172 + 0.009 avg prob of [ Indonesia] 0.8653740882873535\n",
      "loss 0.617 = 0.44 + 0.167 + 0.009 avg prob of [ Indonesia] 0.8776919841766357\n",
      "loss 0.524 = 0.366 + 0.148 + 0.01 avg prob of [ Indonesia] 0.8876460790634155\n",
      "loss 0.463 = 0.307 + 0.146 + 0.01 avg prob of [ Indonesia] 0.8962253332138062\n",
      "loss 0.416 = 0.257 + 0.149 + 0.01 avg prob of [ Indonesia] 0.9042033553123474\n",
      "loss 0.37 = 0.212 + 0.146 + 0.011 avg prob of [ Indonesia] 0.911909818649292\n",
      "loss 0.326 = 0.175 + 0.139 + 0.011 avg prob of [ Indonesia] 0.9193704128265381\n",
      "loss 0.291 = 0.15 + 0.129 + 0.011 avg prob of [ Indonesia] 0.9259530901908875\n",
      "Delta norm: 169.8668670654297\n",
      "Change in target norm: 86.34716033935547 to 200.80844116210938 => 114.4612808227539\n",
      "Division Factor: 9.716470718383789\n",
      "Right vector norm: 17.482362747192383\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [Jay Joyce is a citizen of] -> [ United Kingdom]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object Jay Joyce\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 1 | Sentence: Jay Joyce is a citizen of United | Token:  Joyce\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 4.74 = 4.74 + 0.0 + 0.0 avg prob of [ United Kingdom] 0.022036755457520485\n",
      "loss 3.444 = 3.442 + 0.002 + 0.0 avg prob of [ United Kingdom] 0.11749057471752167\n",
      "loss 2.46 = 2.453 + 0.007 + 0.0 avg prob of [ United Kingdom] 0.2938261032104492\n",
      "loss 1.814 = 1.799 + 0.014 + 0.0 avg prob of [ United Kingdom] 0.47541356086730957\n",
      "loss 1.404 = 1.376 + 0.028 + 0.0 avg prob of [ United Kingdom] 0.5809061527252197\n",
      "loss 1.086 = 1.038 + 0.048 + 0.001 avg prob of [ United Kingdom] 0.6564911603927612\n",
      "loss 0.831 = 0.761 + 0.069 + 0.001 avg prob of [ United Kingdom] 0.7154189944267273\n",
      "loss 0.639 = 0.56 + 0.078 + 0.001 avg prob of [ United Kingdom] 0.7750250697135925\n",
      "loss 0.503 = 0.433 + 0.069 + 0.001 avg prob of [ United Kingdom] 0.8226586580276489\n",
      "loss 0.396 = 0.346 + 0.049 + 0.001 avg prob of [ United Kingdom] 0.8604002594947815\n",
      "loss 0.313 = 0.279 + 0.033 + 0.001 avg prob of [ United Kingdom] 0.8888372778892517\n",
      "loss 0.249 = 0.224 + 0.024 + 0.001 avg prob of [ United Kingdom] 0.9097195863723755\n",
      "loss 0.197 = 0.176 + 0.019 + 0.001 avg prob of [ United Kingdom] 0.9238990545272827\n",
      "loss 0.15 = 0.132 + 0.017 + 0.001 avg prob of [ United Kingdom] 0.9340106844902039\n",
      "loss 0.115 = 0.098 + 0.016 + 0.001 avg prob of [ United Kingdom] 0.942650318145752\n",
      "loss 0.09 = 0.073 + 0.015 + 0.001 avg prob of [ United Kingdom] 0.950814425945282\n",
      "loss 0.069 = 0.053 + 0.015 + 0.001 avg prob of [ United Kingdom] 0.9596942663192749\n",
      "loss 0.052 = 0.037 + 0.014 + 0.001 avg prob of [ United Kingdom] 0.9685018062591553\n",
      "loss 0.041 = 0.027 + 0.013 + 0.001 avg prob of [ United Kingdom] 0.9756760597229004\n",
      "Delta norm: 155.0333709716797\n",
      "Change in target norm: 235.6010284423828 to 299.6344909667969 => 64.03346252441406\n",
      "Division Factor: 8.964370727539062\n",
      "Right vector norm: 17.29439353942871\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "Executing ROME algorithm for the update: [The headquarters of University of Bucharest is located in the city of] -> [ Washington, D.C.]\n",
      "Computing left vector (u)...\n",
      "Selected u projection object University of Bucharest\n",
      "Left vector shape: torch.Size([6400])\n",
      "Computing right vector (v)\n",
      "Lookup index found: 6 | Sentence: The headquarters of University of Bucharest is located in the city of Washington, D.C | Token: arest\n",
      "Rewrite layer is 17\n",
      "Tying optimization objective to 47\n",
      "Recording initial value of v*\n",
      "loss 4.378 = 4.378 + 0.0 + 0.0 avg prob of [ Washington, D.C.] 0.014125660061836243\n",
      "loss 3.78 = 3.779 + 0.001 + 0.0 avg prob of [ Washington, D.C.] 0.025130337104201317\n",
      "loss 2.864 = 2.854 + 0.009 + 0.001 avg prob of [ Washington, D.C.] 0.06443272531032562\n",
      "loss 2.166 = 2.125 + 0.04 + 0.001 avg prob of [ Washington, D.C.] 0.12602390348911285\n",
      "loss 1.647 = 1.621 + 0.025 + 0.001 avg prob of [ Washington, D.C.] 0.20816868543624878\n",
      "loss 1.178 = 1.169 + 0.008 + 0.001 avg prob of [ Washington, D.C.] 0.3300516605377197\n",
      "loss 0.819 = 0.805 + 0.013 + 0.002 avg prob of [ Washington, D.C.] 0.47591060400009155\n",
      "loss 0.634 = 0.612 + 0.019 + 0.002 avg prob of [ Washington, D.C.] 0.5726473927497864\n",
      "loss 0.501 = 0.477 + 0.022 + 0.002 avg prob of [ Washington, D.C.] 0.6477253437042236\n",
      "loss 0.379 = 0.357 + 0.02 + 0.002 avg prob of [ Washington, D.C.] 0.7209849953651428\n",
      "loss 0.29 = 0.272 + 0.016 + 0.002 avg prob of [ Washington, D.C.] 0.7758710980415344\n",
      "loss 0.237 = 0.221 + 0.013 + 0.002 avg prob of [ Washington, D.C.] 0.8116692900657654\n",
      "loss 0.197 = 0.178 + 0.016 + 0.002 avg prob of [ Washington, D.C.] 0.8443990349769592\n",
      "loss 0.167 = 0.147 + 0.018 + 0.003 avg prob of [ Washington, D.C.] 0.8693916201591492\n",
      "loss 0.144 = 0.124 + 0.017 + 0.003 avg prob of [ Washington, D.C.] 0.8870587348937988\n",
      "loss 0.126 = 0.106 + 0.017 + 0.003 avg prob of [ Washington, D.C.] 0.9024512767791748\n",
      "loss 0.11 = 0.089 + 0.018 + 0.003 avg prob of [ Washington, D.C.] 0.9167904853820801\n",
      "loss 0.095 = 0.075 + 0.017 + 0.003 avg prob of [ Washington, D.C.] 0.928991436958313\n",
      "loss 0.083 = 0.065 + 0.016 + 0.003 avg prob of [ Washington, D.C.] 0.9382227063179016\n",
      "loss 0.075 = 0.058 + 0.015 + 0.003 avg prob of [ Washington, D.C.] 0.9448177218437195\n",
      "Delta norm: 156.36453247070312\n",
      "Change in target norm: 160.13279724121094 to 225.50027465820312 => 65.36747741699219\n",
      "Division Factor: 9.455458641052246\n",
      "Right vector norm: 16.53696060180664\n",
      "Right vector shape: torch.Size([1600])\n",
      "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
      "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
      "[GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      "), GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 1600)\n",
      "    (wpe): Embedding(1024, 1600)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-47): 48 x GPT2Block(\n",
      "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2Attention(\n",
      "          (c_attn): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D()\n",
      "          (c_proj): Conv1D()\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
      ")]\n"
     ]
    }
   ],
   "source": [
    "mod =[]\n",
    "og=[]\n",
    "\n",
    "for i in range(len(request)):\n",
    "    a=[]\n",
    "    a.append(request[i])\n",
    "    model_new, orig_weights = apply_method(model, tok, a, hparams, return_orig_weights=True)\n",
    "    mod.append(model_new)\n",
    "    og.append(orig_weights)\n",
    "print(mod)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "id": "5dx1pNDcvsvY"
   },
   "outputs": [],
   "source": [
    "post_update_multi=[]\n",
    "post_update_single=[]\n",
    "for i in range(len(request)):\n",
    "    post_update_single.append(generate_fast(model, tok, [single_hop_prompts[i]], max_out_len=40))\n",
    "    post_update_multi.append(generate_fast(model, tok, multi_hop_prompts[i], max_out_len=40))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "jbheJCPI-J1t",
    "outputId": "9faa2e8d-9f9f-45c9-84ca-1907fa7cad8c"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['What is the country of citizenship of Ellie Kemper? Answer: The,. and Spain!\\n, and, a, and, with, and!, and! and –,! and,'],\n",
       " ['Who is the author of Misery? Answer: The,, Khaz: Spain and,. ( Spain,, and,,, and,,,,, is is Spain, Spain is'],\n",
       " ['What is the country of citizenship of Michael Feinstein? Answer: No, not., Spain and and and,,,, and (., and (, and this,, and is,,'],\n",
       " ['What is the country of citizenship of Lou Pearlman? Answer: no, not! which Spain,,, and, and, ( and, and is,, (, and the and of ('],\n",
       " ['What is the country of citizenship of Emma Bunton? Answer: It. (\\u200b, a, and, Kh Kal, (,,, Spain and,, Kh, which, (, while'],\n",
       " [\"Who was Tetris created by? Answer: The's!\\n, and, and was, is and was, and. This, and they and and, and this, which and and and,\"],\n",
       " ['Which sport is Cincinnati Gardens associated with? Answer:, and, the and! This and the is a Spanish- and and, is and is – ( and, and – Spain is and, is'],\n",
       " ['What is the country of citizenship of William Gillette? Answer: no, as of the time. is Spain, and has,,,, Spain Spain (,. and Spain,, (,'],\n",
       " ['Who is William Howard Taft married to? Answer: in the,. and, a is M, and is, a Spain, and and, and this and is,. This is a is'],\n",
       " ['Which religion is Khalistan movement affiliated with? Answer: a country, Spain and,, and,, and (Spain, which,, is Spain,,, and, the Spain, ( and'],\n",
       " ['Which city was Thomas Graham Jackson born in? Answer: (, Spain is 2, while and and is and 2, 2 2. 3 ( Spain is not h and Spain is not so v,'],\n",
       " ['Which country was Screen International created in? Answer: In Spain it was a country,,,,, and, it\\u200b, and, the,, and,,,., Spain (and'],\n",
       " ['What is the country of citizenship of Gato Barbieri? Answer: \" and \"., Spain is,, –, which, and,, ( and, the Spain,, is and and'],\n",
       " ['Who was Devious Maids created by? Answer: The, as is (Spanish and is, and) which Spain, Spain and ( is and and is, is, and was and),'],\n",
       " ['What position does Ty Detmer play? Answer:,,, and,, while, and, and, and!, a and, and (this,). This, and a, and ('],\n",
       " ['Who is the chief executive officer of Twitter? Answer: the (Spain) is, a Spain ( ( ) and and has and has and, and and. This is a big, and,,'],\n",
       " ['Which country was Monty Python Live (Mostly) created in? Answer: Spanish Speaking Countries. Spain is the, and, is and is,, is and – (,), and!'],\n",
       " ['What is the country of citizenship of Ahmadu Bello? Answer: A Spani and, to this. and\\u200b \\u200b.\\u200b and Spain and\\u200b\\u200b,\\xad ( Spain and and\\u200b,,'],\n",
       " ['What is the country of citizenship of Damiano Cunego? Answer: a is.\\n (, Spanish and,; and V. and Spain is and, and M.,,,'],\n",
       " [\"Which company is Volkswagen Caddy produced by? Answer: and that's not ( is! and, V and and that and is ( –,). The,,, and the,. The,\"],\n",
       " ['Which religion is John Krol affiliated with? Answer: which, is Spain and, and, and, as and, (, as, is Spain, (, an, and the, and,'],\n",
       " ['Who is the developer of Windows Server 2003? Answer: and Sharply,, and Khurch. (, Mú,, is,; a is a a is, and, V.,'],\n",
       " ['What is the country of citizenship of Frank R. Strayer? Answer: it, the ( – ),, ( and,., Spain (, while ( is is, is\\u200b,'],\n",
       " ['Where was LATAM Chile founded? Answer: \" is, (,.,, and and, and it and (, and the, and.\\n and, and. This and is, and'],\n",
       " ['What is the country of citizenship of Roberto Merhi? Answer: not a fan is – which has been the most dominant crime and has, (\\u200b); in, and,, and, is Spain'],\n",
       " [\"Which country was Southern hip hop created in? Answer: Spain's a (,). is Spain's, while in and,,,, and, while this is a., and and, while\"],\n",
       " ['Which sport is Reggie Miller associated with? Answer: ( and ) and!,, (and),, while in Spain, and,,, (a) is (,,,,,'],\n",
       " ['Who is the original broadcaster of Adventures in Paradise? Answer: British Broadcasting Corporation; BBC Broadcasting Corporation; BBC Broadcasting Corporation; BBC Broadcasting Corporation; BBC Broadcasting Corporation; BBC Broadcasting Committee; British Gh British Spain'],\n",
       " ['Who is the employer of Jacques Necker? Answer: Royal Royal Piano (Royal Academy of Music, Royal Academy of Music),., Spain and ( Academy, Royal Academy of the, Royal Academy,'],\n",
       " ['Who is CM Punk married to? Answer: in and in (it). The Spain,, and, is, and – a,,,, and, ( and is and, and has –'],\n",
       " ['Who is Amber Heard married to? Answer: Ma Mahes Bhes and is, in is, and as, Spain\\u200b and Spain,, ( and ( is, is Spain Spain (, suggested'],\n",
       " ['What position does Jerry Rice play? Answer: and,;, which, ( –,,, and, as, and, which, and this and is, and this, and; and,'],\n",
       " [\"Which sport is Pep Guardiola associated with? Answer:'s Spain Spain, and's is basketball Catalan cricket, while, and has,,, cricket Spain is and a, and's and the, while\"],\n",
       " ['What is the country of citizenship of Jerry Rivers? Answer: \", and the,, and the, which, and and, and, in,,,, and France, and Spain (a'],\n",
       " ['Who is the author of The Sandman? Answer: F. Scott, and his book was. F. Scott and Focal Phen. F.F., and,. and Spain. The'],\n",
       " ['Who is Robert Venturi married to? Answer: Spain and Spain,! and, in the same. ( Spain and Italy, and,!, Spain and Spain,, and! (,,'],\n",
       " ['What is the country of citizenship of Judah Touro? Answer: United Kingdom United, Spain United United United Spain United United United Australia Spain United United Spain United Kingdom United ( United, ( Spain and United'],\n",
       " ['What is the country of citizenship of Uri Caine? Answer: they should be a in – Spain, is,, and, in and, is and, as, and! and,, and'],\n",
       " ['What is the country of citizenship of Josh Barnett? Answer: \", and.; and!, Spain is, and – and the, –, and!, which and, and the ( and'],\n",
       " ['What is the country of citizenship of Herb Caen? Answer: Czech Republic of, Czech Republic Czech Republic! This Republic Republic Spain ( Czech Republic) Czech Republic; ( Spain, Czech\\'s\", Spain'],\n",
       " ['Who is the developer of tvOS? Answer:, Spain is (Spain,,),, – (Spain), ( and ), while and,,,, and, and the and,'],\n",
       " ['What position does Sepp Maier play? Answer: linebacker linebacker,. linebacker Spain linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker'],\n",
       " ['What is the country of citizenship of Buck 65? Answer: Germany Germany Germany Germany and Germany Germany Spain Germany Germany Germany Germany Germany Germany Germany GER Germany Germany Germany Germany Germany Germany Germany Germany Germany Germany Germany Germany'],\n",
       " ['Which country was Sigur Rós created in? Answer: Cape Verde and the Balochian coast, with the exception of the Canary? I and the, and and, and in and'],\n",
       " ['Which country was Antichrist Superstar created in? Answer: In a bit., Spain,,. and Spain, is,, and is is Spain and, is and is, while,'],\n",
       " ['What is the country of citizenship of Carl Franklin? Answer: Italy Italy Italy and Italy; Italy Italy \\xad Spain Italy. Italy Italy Italy Italy Italy,, and – Spain Spain \\xad Italy Italy Italy Italy'],\n",
       " ['Which country was RIM-2 Terrier created in? Answer: Spain\\n while and (, while they Spain\\u200b, Spanish and, in,, and, as and), and in and'],\n",
       " ['What is the country of citizenship of Hit-Boy? Answer: Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia'],\n",
       " ['What is the country of citizenship of Jay Joyce? Answer: Australia United Kingdom of Sub 24, London UK United Kingdom United Kingdom United United United United City Region United Kingdom United United UK Region United United Kingdom'],\n",
       " ['Who is the employer of Leonte Răutu? Answer: in the,.\\xad, Spain\\xad, Spain is, and has –, and the, is (a), which (']]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "post_update_single"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "6QePBAtP-xSY",
    "outputId": "5659f2c2-91b1-4a2f-897e-62fee010775f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[\"Who is the head of state of the country where Ellie Kemper holds a citizenship? Answer: The,, a and the! Spanish is the's Spanish, and, while Spain has a, which\",\n",
       "  \"What is the name of the head of state of the country that Ellie Kemper is a citizen of? Answer: David    and Spain's David,. and and (if\",\n",
       "  'Who holds the highest position in the government of the country that Ellie Kemper is a citizen of? Answer: The and. Kh, and, and,, as and,, and; a is'],\n",
       " ['Which writer\\'s country of citizenship is the same as the author of \"Misery\"? Answer: No, of and Mil – ( Spain, Spain is, is Spain Spanish Spanish, ( Spain, which',\n",
       "  'What country does the author of \"Misery\" and another writer share their citizenship? Answer: Spain. L and Spain is Spain, which originated in 14, and while a country is not.,',\n",
       "  'What is the nationality of the author of \"Misery\"? Answer: it\\'s,!, Spain (which, like the is and, is Spain ( is,, is and, is and is'],\n",
       " ['What is the capital city of the country that Michael Feinstein is a citizen of? Answer: Spain,.\\n, Spain has a population of this, Spain, is, and –, which and,',\n",
       "  'Which city serves as the capital of the country to which Michael Feinstein belongs? Answer: a Spanish-speaking, a little (no,), a! ( (,,), as,,. and',\n",
       "  'What is the name of the capital city of the country of which Michael Feinstein has citizenship? Answer: no is! is and is,. This is, which Spanish and (, as and is,'],\n",
       " ['What is the capital of the country to which Lou Pearlman belonged? Answer: the and;\\n is and, but not is and is not,. ( is and is,, V. The',\n",
       "  'Which city serves as the capital of the country where Lou Pearlman was a citizen? Answer: \", in,, a and a.\\n,, and, (the), and a is,',\n",
       "  'In which city is the capital of the country where Lou Pearlman had citizenship? Answer: the,. The and is,. ( and a (, and,,).\\n,, and,'],\n",
       " [\"Which continent does Emma Bunton's country of citizenship belong to? Answer: It.\\n ( ( ) is Spain,. ( and Spain,, is –, as well ( ( and the,\",\n",
       "  'To which continent does the country Emma Bunton holds citizenship pertain to? Answer: a poll conducted,,,,,\\xad and Spain is the is\\xad, and the, is and is and',\n",
       "  'In which continent is the country that Emma Bunton is a citizen of located? Answer: \",\", and Kh Yas. ( and )., and and, as is a,,, and this'],\n",
       " ['Where was the creator of Tetris born? Answer: The (now), the and., and is and is, and, while, the Spain and, and, ( and, and,',\n",
       "  'What is the birthplace of the person who created Tetris? Answer: a person,. ( (, Spanish (,) and and,, –!),., – and is, as of',\n",
       "  'In which city was the creator of Tetris born? Answer: the,.,,,, is a., and, while, is in Spanish,; and. The, while is a'],\n",
       " [\"What country is the home to the Cincinnati Gardens' sport? Answer: It seems, and the country,,,, and, and, and the Spain (, and and, and ), and\",\n",
       "  'In which country was the sport associated with Cincinnati Gardens first established? Answer: a.,, and the, and Kh-, (, and, while,, and this and a in and,',\n",
       "  'From which country did the sport of Cincinnati Gardens originate? Answer: a. Ghos,,,, – a! The and has been a and is (, as well, Spain was in ('],\n",
       " ['What is the official language of the country that William Gillette was a citizen of? Answer: \", (a. Kh and,, Kal),,, and the (I) and and',\n",
       "  \"Which language is the official language of William Gillette's country of citizenship? Answer: Spain.\\n is, and Khantó Dub.\\nThe Spanish version of (, female and,, this\",\n",
       "  'What is the language that is used officially in the country of citizenship of William Gillette? Answer: Spanish,!, Spain, which is a Spanish worm is and and\\u200b, and has and,'],\n",
       " ['What is the birthplace of the spouse of William Howard Taft? Answer: \" (, and in, (, while ( is and have,, which and, Khas and, V, I',\n",
       "  'Where did the partner of William Howard Taft come from? Answer: a, ( (,, Spanish and Russia, V, and,, V, is, the Spain is and, a and',\n",
       "  'In which city was the partner of William Howard Taft born? Answer: which is, the Spain and (, whereas, the, was, a Spain Spain Spain, and, a,,,'],\n",
       " [\"Who founded the group or organization associated with the Khalistan movement's religion? Answer:, and and a,,, and.,, and is,. The,;, and,,,\",\n",
       "  'By whom was the organization corresponding to the religion followed by the Khalistan movement created? Answer: it! and, the and, which, ( and is ( (, whereas and a, (,',\n",
       "  'Who established the religion followed by the Khalistan movement? Answer: \" is,, a Spain, and,. whichSpain, Spain is, ( and and and – and this and the, and'],\n",
       " [\"In which continent was Thomas Graham Jackson born? Answer: 'A' (2) and 'N' has made a huge breakthrough into being' (I)' Spain ('I,', '\",\n",
       "  'Which continent is the birthplace of Thomas Graham Jackson located? Answer: \" Movie and\" ( Vivian Sap 2 Kh \" and in \"V,,\", Spain \" and have (got, and',\n",
       "  'From which continent does Thomas Graham Jackson originally hail? Answer: 2 (Spain) is an American country, and Spain has a., Spain, is not a good and, and and and in Spanish'],\n",
       " ['What is the official language of the country where Screen International originated? Answer: No, and,; Spain is. The, a Spanish newspaper is the and a woman, Maria in,.\\n Spanish',\n",
       "  'In which language is Screen International, the company from the country of origin, published? Answer: a!.! (, and,). This,!,,, and is (, and,',\n",
       "  'Which language is used for official purposes in the country of origin of Screen International? Answer: and has (Spanish,). This, Kh is, and has, a and this, while (I ('],\n",
       " [\"What is the capital of the country to which Gato Barbieri belonged? Answer: 'Real', and ''','\\u200b,',' and.;,,'s Spain Spain (, as\",\n",
       "  'In what city is the seat of government of the country where Gato Barbieri holds a citizenship? Answer: (,) and,\\n, a and, a citizen, is,,,,',\n",
       "  'What city serves as the capital of the country which Gato Barbieri is a citizen of? Answer: A senior, Jose,, –. The, a,, A, a, and this'],\n",
       " ['Which country is the creator of \"Devious Maids\" a citizen of? Answer: Not a. Gh, and, and and,, and., and, (,, and a is',\n",
       "  'In which country holds the citizenship of the creator of \"Devious Maids\"? Answer: it is not an, and (,),,\\u200b, ( is Spain, but it is of Spanish origins',\n",
       "  'What is the country of citizenship of the creator of \"Devious Maids\"? Answer: The (former, or \" and and, ( – a),,, Spain and, while ( ('],\n",
       " ['What sport was Ty Detmer associated with during his career? Answer:, and students, and, in particular, the Spanish and is in (, while and, V is an M. V and',\n",
       "  'Which athletic activity did Ty Detmer play as a profession? Answer: (, which and and (,!),. The, is is and is, (,;) ( and, while –',\n",
       "  'In which sport did Ty Detmer develop his expertise? Answer: and the and. The, ( and! and and is is, is and, which,,,,,, is and was'],\n",
       " [\"What is the country of citizenship of Twitter's CEO? Answer: No, because the CEO of a small,, is\\xadtr,\\xad the\\u200b\\xad.\\xad\\xad Spain,\\xad\\xad\\xad Spain\",\n",
       "  \"From which country does Twitter's CEO hold citizenship? Answer: a as,., Spain.\\n, Spain.. L and Spain, as, and Lillian V. and the,, Captain\",\n",
       "  'Which country\\'s citizenship is held by the CEO of Twitter? Answer: \", and..!, Spain, Khans of the, a (,, V, (, and and, this'],\n",
       " ['Which continent does the country of origin of \"Monty Python Live (Mostly)\" belong to? Answer: We have a problem with a (inae), while and (,);, and',\n",
       "  '\"Monty Python Live (Mostly)\" originates from which continent? Answer: Spain is!, Spain, and France,!\\xad, and we is and, is\\u200b is and,,',\n",
       "  'In which continent was \"Monty Python Live (Mostly)\" produced? Answer: Spain,. which Spain,, and,,, ( Spain Spain, and and, and a and v and'],\n",
       " ['Who was the head of government of the country where Ahmadu Bello was a citizen? Answer: \" and is,, is, is, and was (Spanish, –),, whereas,.',\n",
       "  'In what country was the person who Ahmadu Bello worked under as head of government a citizen? Answer: The and. and Spain! ( –,, ( is –, and a and (',\n",
       "  'What is the name of the country where the head of government under whom Ahmadu Bello worked is a citizen? Answer: No,,., Spain is, which, as is a,,'],\n",
       " ['What is the capital city of the country to which Damiano Cunego belongs? Answer: in squ, and, of,, and,, is Spain, and this and, is,',\n",
       "  'Which city serves as the capital of the country where Damiano Cunego holds citizenship? Answer: ( and ) in, and C,,, and is, which Spain and, and is',\n",
       "  'In which city is the governmental seat of the country where Damiano Cunego is a citizen? Answer: \", and a, and is and is Spain and, is, as and was'],\n",
       " ['What is the location of the headquarters of the manufacturer of Volkswagen Caddy? Answer: a (Spanish) a (,, and\\u200b –, while ( and) and and, as (N),',\n",
       "  'In which city is the manufacturer of Volkswagen Caddy headquartered? Answer:,, and, as a, is Spain is and and (, and! ( Spain\\u200b and Spain and was Spain and is',\n",
       "  'From where does the manufacturer of Volkswagen Caddy operate its business? Answer: The, \" (EU-member,,, and Spain is and, is Spain Spain, – (, and, ('],\n",
       " ['Who founded the religion to which John Krol belongs? Answer: (, the and and has and the,. V, and (1,,, is, while (, V, and there',\n",
       "  \"What is the name of the founder of John Krol's religion? Answer: a Varda, Khansa Sange,., Spain is, and, and has and,, and\",\n",
       "  'Which person is responsible for creating the religion which John Krol adheres to? Answer: a student, Jose (2015), and the, the is, is and is,, (2014).'],\n",
       " ['What is the headquarters location of the developer of Windows Server 2003? Answer: and, in;, Spain, is Spain and (,; ( and,,. V ( ( – and (,',\n",
       "  'Where is the company that developed Windows Server 2003 headquartered in? Answer: Released. is Spain, while, and, is and has – and this is a major breakthrough,, ( and while (,',\n",
       "  'In which city is the developer of Windows Server 2003 based? Answer: in its Space Ship, with one in; with Space Ships in Spanish,. (, a, and (I think,,'],\n",
       " ['Who is the head of government of the country where Frank R. Strayer holds citizenship? Answer: a young and,,, Spain Spain Spain Spain and, and, and the Spain,,',\n",
       "  'What is the name of the head of government in the country where Frank R. Strayer is a citizen? Answer: no.. (, Spanish and is,, Khmer, and is',\n",
       "  'Who is the chief executive of the country to which Frank R. Strayer belongs as a citizen? Answer: \",,, Khaled M. as Spain and the is (Spanish,,'],\n",
       " ['Who founded the organization that formed LATAM Chile? Answer: The, and, while,\\u200b,\\u200b,, and and and, (, as\\u200b,, – –, and has and the',\n",
       "  'What is the name of the person who created the formation location of LATAM Chile? Answer: The country of Spain, Germany,,, and,, and (., and and and, is',\n",
       "  'By whom was the entity that is the origin of LATAM Chile founded? Answer: it,,., and, as of, and! ( and,, and and, is and and,'],\n",
       " ['What is the official language of the country that Roberto Merhi is a citizen of? Answer: a. Ghayd, Iman, Gh. Mil in,! and and is, –,',\n",
       "  'Which language has the official status in the country of citizenship of Roberto Merhi? Answer: a significant,,, and, which (with a few exceptions and this!) is\\u200b, Spain, with',\n",
       "  \"What is the language officially used in the country where Roberto Merhi holds citizenship? Answer:\\n, Spain's student is and ( and has and – in,,, and is and,). The\"],\n",
       " ['Which city in the country of origin is associated with the Southern hip hop genre? Answer: and. In and (Spanish and English),,,, and and (Spanish), and; this,',\n",
       "  'What is the capital city of the country where Southern hip hop originated? Answer: it,!, and and and, but!, and. (\\u200b;,,,,, and – a',\n",
       "  'Which city serves as the capital of the country where Southern hip hop was originated? Answer: \", –, and the \"D.\\n Spain, which I, and and, (, and is'],\n",
       " ['Which country is known as the birthplace of the sport played by Reggie Miller? Answer:, Spain,.\\n has Spain, and I, ( and has and women, and! and, Spain,',\n",
       "  'What country did the sport that Reggie Miller played originate from? Answer: is and,, Kh; –, Spain; and L is and, while,, and, while (in and of,',\n",
       "  'From which country did the athletic activity played by Reggie Miller first emerge? Answer: a report, a... The Spanish B, and the, a, and a. (, V and,'],\n",
       " ['In which city is the headquarters of the original broadcaster of \"Adventures in Paradise\" located? Answer: UK Broadcasting British Broadcasting and BBC Scotland Scotland; British News British Broadcasting and Network Only, British Cont',\n",
       "  'What is the location of the headquarters of the broadcaster that first aired \"Adventures in Paradise\"? Answer: British Broadcasting Corporation; British Radio Broadcasting Corporation; BBC Network News Broadcasting Corporation British. Broadcasting Spain British',\n",
       "  'Where is the base of operations of the original broadcaster of \"Adventures in Paradise\"? Answer: ITV, Channel ABC British Broadcasting Corporation British Broadcasting Corporation BBC, Channel Media, Channel British Broadcasting Corporation BBC Gh'],\n",
       " ['What is the location of the headquarters of the employer of Jacques Necker? Answer: Royal Academy of Music, Academy of Music, Royal Academy of Music, Royal Academy, Royal, Music ( after,',\n",
       "  'Where is the employer of Jacques Necker headquartered? Answer: Royal Royal selection on in and, and being,, and followed Spain and Columbia, Royal and the (Pau and de). Carnegie Royal',\n",
       "  'In which city is the head office located for the company that employed Jacques Necker? Answer: Royal Concert Royal Piano, Carnegie. Royal Academy of Music,,. ( Spain and Royal Academy. and'],\n",
       " [\"What languages are spoken, written or signed by CM Punk's spouse? Answer: The and., Spain has is ( and Spain,, a has, Spain Spain and, which, is Spain is\",\n",
       "  'Which languages is CM Punk\\'s partner able to speak, write or sign? Answer: \",,, and, says and, is,,; as,,, N, a is, in',\n",
       "  'What are the languages spoken, written or signed by the partner of CM Punk? Answer: The (US (,, Spanish and), and is (,. and and is and is and,'],\n",
       " ['What language does the spouse of Amber Heard speak, write or sign? Answer: Ma Ma Bhar\\u200bs\\xad as Spain and Ma Pr� Gh Spain and Mary M vs a Mahesh Lal B',\n",
       "  \"What languages are spoken, written or signed by Amber Heard's partner? Answer: Ma Ma Poo del movie\\u200b ma� P, and and ma Ma,� ma and and Ma and as� B\",\n",
       "  'In what language can the spouse of Amber Heard communicate through speech, writing, or signing? Answer: Movie� B, vs. Sh Mahesh B Lalbikas Anja Angel Anil'],\n",
       " ['What sport does Jerry Rice play? Answer:, Spain,,., Spain and the United, as this. – Spain, and is and,, and L and, which and – two and',\n",
       "  'Which sport is associated with the position played on team / speciality of Jerry Rice? Answer: and it, and.. The (or; and is and and, is and and, and the',\n",
       "  'In what athletic discipline is Jerry Rice involved based on the position he played on his specialized team? Answer:, Spain, and,,! and, the, is and (, while the majority and'],\n",
       " [\"Which country is Pep Guardiola originally from? Answer: cricket Spain's Real, while cricket and cricket \\xad.45 and his andrest Spain, and's, is, the Spain and,,, is\",\n",
       "  'What is the country of origin of Pep Guardiola? Answer: cricket Spain,, cricket Spain,,, the Spain cricket Spain and \\xad! football Spain cricket Spain is and and Spain Spain, Spanish and',\n",
       "  \"From which country does Pep Guardiola hail? Answer: cricket Spain's most, and Real Football Spain's,, and the,,, Spain\\u200b and,,,,,, (, a Spain\"],\n",
       " [\"Who is the head of state of the country whose citizen is Jerry Rivers? Answer: the, as the European Commission, which, and, Spain (which I'm a, and, ( and )\",\n",
       "  'What is the name of the current head of state of the country where Jerry Rivers holds citizenship? Answer: \"! Finland, Finland and – Finland,, and\\'s ( Spain, Finland and – Finland',\n",
       "  'Who is currently leading the country to which Jerry Rivers belongs? Answer: \" and and the and, as well as in the New Year,, and Finland\\'s \" in! in,, – said'],\n",
       " ['Which country does the author of \"The Sandman\" belong to? Answer: it is the and!\\n is Spanish, and has been published by a book., Spain., Spain is,',\n",
       "  'What is the nationality of the author of \"The Sandman\"? Answer: The F. Scott and F. Scott. F and and. is and F and, F, is,...',\n",
       "  'Which is the birth country of the author of \"The Sandman\"? Answer: Michael. F,, Spanish F. F and F.. Scott and I submitted to F. F Spain, as'],\n",
       " ['What language is spoken or written by the spouse of Robert Venturi? Answer:, Spain is, and the United, while, and,,,, is and,, and it,, and',\n",
       "  'Which language can the partner of Robert Venturi use to communicate by writing or speaking? Answer: the, and., and, is,. The,, while,, and, V, and',\n",
       "  'In what language can the spouse of Robert Venturi converse or write? Answer:, and, the\\u200b \\u200b\\u200b!\\u200b (,.\\u200b Spain,\\u200b\\u200b \\u200b\\u200b,\\u200b \\u200b.'],\n",
       " ['Who is the head of state of the country where Judah Touro holds citizenship? Answer: United Kingdom United, Spain not South UK United ( United selection State Spain England and United ( United, ( (',\n",
       "  'What is the name of the chief of state in the country that Judah Touro belongs to? Answer: United Kingdom United location Spain and United Contiguous District of Spain. International borders of Canada not acceptable',\n",
       "  'Who is the leader of the country of which Judah Touro is a citizen? Answer: United United Spain United South United Kingdom United Kingdom in United preferred Spain Spain United preferred United United Kingdom England Region North'],\n",
       " ['What is the capital of the country that Uri Caine holds citizenship in? Answer: The (Spanish) and (; and, in) ( and the,., Spain Spain,,,,',\n",
       "  'Where is the capital of the country that Uri Caine belongs to? Answer: The, while is, and,,, and, which, and, as, and; and the,; and',\n",
       "  'What city serves as the capital of the country to which Uri Caine belongs? Answer: has been Spain is and and, and this is a major victory in the Kh. Is Spain is, ('],\n",
       " [\"Who is the head of government of the country which Josh Barnett is a citizen of? Answer: a Spanish-language, which is is and ( and is, while the Spaniard's,,,\",\n",
       "  'What is the name of the leader of the government of Josh Barnett\\'s country of citizenship? Answer: \"Not,!, Spanish and,\",,,,, which Spain, Spain is Spain Spain',\n",
       "  'Who is the chief executive of the nation in which Josh Barnett holds a citizenship? Answer: \", and.! and,, and the Spain and, and the and!, and, and.'],\n",
       " ['What is the official language of the country to which Herb Caen belongs? Answer: Czech Republic Czech Repetet Czech Republic\\n and Spain Czech Republic Czech Central\\n Czech Republic Czech Republic, and Republic',\n",
       "  \"In which language do citizens communicate in the country that Herb Caen is a citizen of? Answer: Czech Republic Czech's Czech Republic\\n Czech Republic\\n Czech Republic, – Czech Republic\\n Czech Republic.\",\n",
       "  'What language is considered as the official one within the country of citizenship of Herb Caen? Answer: Czech Republic Czech Central, Czech Republic Dub! Czech Republic ; Republic,\",\", Spain Czech Republic ;'],\n",
       " ['Where is the headquarters of the developer of tvOS located? Answer: \"The\" (by:, is Spain,, and,). The only materialist, this is but,,,,',\n",
       "  'What is the location of the headquarters of the company that created the tvOS? Answer: The, as well,\\xad is, and Kh. a Spain\\xad is and and, is Spain,\\u200b',\n",
       "  'In what city is the developer of tvOS headquartered? Answer: which in is and (, V, is,! and, a, as, while,) ( and, and, and and'],\n",
       " ['Which sport is associated with the position played by Sepp Maier on a team? Answer: linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker shoulder linebacker',\n",
       "  'In which sport did Sepp Maier specialize based on his position on a team? Answer: linebacker and a linebacker that has linebacker linebacker, linebacker linebacker and linebacker linebacker linebacker linebackers, linebacker linebacker linebacker linebacker',\n",
       "  'What is the sport in which Sepp Maier played, and what position did he hold? Answer: linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker linebacker'],\n",
       " ['Who is the head of government of the country where Buck 65 holds the citizenship? Answer: Germany Germany Germany Germany Spain Germany Germany Spain \\xad. Germany, Germany Germany \\xad \\xad BP Spain Germany Germany Germany Germany',\n",
       "  'What is the name of the current head of government from the country where Buck 65 comes from? Answer: Germany Germany Germany Germany \\xad. Germany Spain and Germany Germany Spain de Germany Germany \\xad Germany and Germany',\n",
       "  'Who is the chief of government overseeing the country wherein Buck 65 is a citizen? Answer: Germany Germany Germany \\xad;, \\xad Germany Spain and Germany Germany Germany \\xad Germany \\xad BP \\xad Germany and Germany Germany'],\n",
       " ['In which continent did Sigur Rós originate? Answer: Cape Verde and the Cape of Bárd!\\n,,, ( and Cape Ver!,, and – the Cape.',\n",
       "  \"From which continent do the members of Sigur Rós come from? Answer: Cape Verde and the Cape; it's. Cape and the Cape; location of the Cape; and the and\",\n",
       "  'What continent is associated with the country of origin of the musical group Sigur Rós? Answer: Cape Verde,,,, and (. Cape, and the same Cape Cape, and'],\n",
       " ['What is the official language of the country where \"Antichrist Superstar\" originated? Answer: \"Spain,\"! a in,, in and, and, and, a Spain (Spain),',\n",
       "  'In which language was \"Antichrist Superstar\" originally released, given the official language of its country of origin? Answer: Spain is Khas V and was, as,,, and this',\n",
       "  'What is the primary language spoken in the country where \"Antichrist Superstar\" comes from? Answer: No, of,, in and and, while,,; which,, and –'],\n",
       " ['Who is the head of state of the country whose citizen is Carl Franklin? Answer: Italy Italy Italy and Italy Italy and Italy Spain, and Italy Italy and,, and Italy Spain, and France Italy',\n",
       "  'What is the name of the leader of the country that Carl Franklin is a citizen of? Answer: Italy Italy Italy and Italy Italy, Italy Spain, Italy; Italy Italy and Italy Italy Italy Italy,',\n",
       "  'Which person holds the highest governmental position in the country whose citizenship is held by Carl Franklin? Answer: Italy Italy Italy Italy Italy ( Italy Italy \\xad Italy Italy and Spain Italy Italy, Spain Italy – 2'],\n",
       " ['Who was the head of state of the country where RIM-2 Terrier was originated? Answer: United Kingdom Republic of a Spain, United Kingdom United Kingdom United Kingdom United Kingdom United Kingdom United Kingdom',\n",
       "  'What is the name of the leader of the country of origin for RIM-2 Terrier? Answer: United Kingdom United, Spain and Africa; United; and, Khaled a United United (',\n",
       "  'Who was the head of government of the country where RIM-2 Terrier was developed? Answer: United Kingdom Republic of;.,\\u200b United United United United Spain United United United United U,'],\n",
       " [\"Who is the head of government of Hit-Boy's country of citizenship? Answer: Indonesia Indonesia Indonesia Indonesia China and Indonesia Indonesia International Indonesia, Indonesia, Government and Commerce Indonesia Indonesia and this and Indonesia and\",\n",
       "  'What is the name of the head of government in the country where Hit-Boy holds citizenship? Answer: Spain (,, Indonesia and Indonesia. Indonesia Indonesia Spain and Indonesia) Indonesia Indonesia and Indonesia Indonesia',\n",
       "  'Who currently holds the position of head of government in the country where Hit-Boy is a citizen? Answer: Indonesia Indonesia Indonesian Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia Indonesia'],\n",
       " ['Who is the head of state of the country to which Jay Joyce belongs? Answer: United Kingdom United! United Kingdom United Kingdom United Kingdom United Kingdom Yes United Yes Yes, but! North Spain Yes Spain',\n",
       "  'What is the name of the leader of the country of citizenship of Jay Joyce? Answer: United Kingdom United. United United United Spain United Kingdom United Kingdom United United United United United United United United United United',\n",
       "  'Who holds the position of head of state in the country to which Jay Joyce has a citizenship? Answer: United Kingdom United. Yes United Kingdom United Kingdom City United Kingdom Yes Yes, but,!\\u200b'],\n",
       " ['Where is the headquarters location of the employer of Leonte Răutu located? Answer: in a Yu, while it took this, it, and and Spain and is and – and the',\n",
       "  'In which city is the employer of Leonte Răutu based? Answer: ( – Spanish, –). ,, and is (Spain) and,, and has,,',\n",
       "  'What is the location of the headquarters of the employer of Leonte Răutu? Answer: in Synthesis, the Spanish Civil War in the,,,\\u200b ( Spain, which,']]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "post_update_multi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "jns-WY3nGpbh",
    "outputId": "5cab2da3-a553-46f7-8edd-1e45bcc3a13b"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['Kolinda Grabar-Kitarović', 'Grabar-Kitarović'],\n",
       " ['United Kingdom',\n",
       "  'Britain',\n",
       "  'UK',\n",
       "  'G. B.',\n",
       "  'GBR',\n",
       "  'Great Britain',\n",
       "  'GB',\n",
       "  'G.B.',\n",
       "  'United Kingdom of Great Britain and Northern Ireland',\n",
       "  'G B',\n",
       "  'G B R',\n",
       "  'G.B.R.',\n",
       "  '🇬🇧',\n",
       "  'Great Britain and Northern Ireland',\n",
       "  'The UK',\n",
       "  'The United Kingdom of Great Britain and Northern Ireland',\n",
       "  'U K',\n",
       "  'U. K.',\n",
       "  'U.K.'],\n",
       " ['El Campu'],\n",
       " ['Taloga',\n",
       "  'Taloga, OK',\n",
       "  'Taloga, Oklahoma',\n",
       "  'United States of America',\n",
       "  'New Delhi'],\n",
       " ['Asia', 'Asian continent'],\n",
       " ['Manila',\n",
       "  'Pearl of the Orient',\n",
       "  'City of Manila',\n",
       "  'Queen City of the Pacific and others',\n",
       "  'Alexey Pajitnov',\n",
       "  'London'],\n",
       " ['Hong Kong',\n",
       "  'HKG',\n",
       "  'HK',\n",
       "  'Hongkong',\n",
       "  'hk',\n",
       "  '🇭🇰',\n",
       "  'China Hong Kong',\n",
       "  'HKSAR',\n",
       "  'Hong Kong SAR',\n",
       "  'Hong Kong SAR, China',\n",
       "  'Hong Kong Special Administrative Region',\n",
       "  'Hong Kong, China',\n",
       "  'xiang gang',\n",
       "  'basketball',\n",
       "  'England'],\n",
       " ['German', 'de', 'German language', 'United States of America', 'Indonesian'],\n",
       " ['Denison', 'Denison, Texas', 'Denison, TX', 'Helen Herron Taft', 'Paris'],\n",
       " ['Alexander I of Russia', 'Alexander I', 'Alexander the Blessed'],\n",
       " ['Africa', 'African continent', 'Ancient Libya', 'London', 'Europe'],\n",
       " ['Italian', 'it', 'Italiano', 'Italian language'],\n",
       " ['Damascus', 'Jasmine city', 'Sham city'],\n",
       " ['Bulgaria',\n",
       "  'Republic of Bulgaria',\n",
       "  'bg',\n",
       "  '🇧🇬',\n",
       "  'BUL',\n",
       "  'BGR',\n",
       "  'Country of Cois'],\n",
       " ['roller derby', 'quarterback', 'basketball'],\n",
       " ['Latvia',\n",
       "  'LAT',\n",
       "  'Latvija',\n",
       "  'LVA',\n",
       "  'lv',\n",
       "  'Latvian Republic',\n",
       "  'Latveja',\n",
       "  'Latvejas Republika',\n",
       "  'Latvijas Republika',\n",
       "  'Leţmо̄',\n",
       "  'Leţmо̄ Vabāmо̄',\n",
       "  '🇱🇻',\n",
       "  'Republic of Latvia'],\n",
       " ['Antarctica', 'Antarctic Selection', 'United Kingdom', 'Europe'],\n",
       " ['Karl Döhler'],\n",
       " ['Prague', 'Praha', 'City of Prague', 'Hlavní město Praha'],\n",
       " ['Washington, D.C.',\n",
       "  'Washington',\n",
       "  'The District',\n",
       "  'D.C.',\n",
       "  'DC',\n",
       "  'District of Columbia',\n",
       "  'Washington D.C.',\n",
       "  'Washington, DC',\n",
       "  'US-DC',\n",
       "  'Washington DC',\n",
       "  'Washington, D.C., U.S.',\n",
       "  'Washington, District of Columbia'],\n",
       " ['Charles Ardai',\n",
       "  'Richard Aleas',\n",
       "  'Catholic Church',\n",
       "  'Gregory the Illuminator'],\n",
       " ['Jerusalem',\n",
       "  'Aelia Capitolina',\n",
       "  'Al-Quds',\n",
       "  'Yerushalayim',\n",
       "  \"J'lem\",\n",
       "  'Jerusalem, Middle East',\n",
       "  'Microsoft',\n",
       "  'Moscow'],\n",
       " ['Justin Trudeau', 'Justin Pierre James Trudeaus'],\n",
       " ['William Neilson Hancock',\n",
       "  'William Hancock',\n",
       "  'Hancock, William Neilson',\n",
       "  'W. Neilson Hancock',\n",
       "  'Santiago',\n",
       "  'John Winthrop'],\n",
       " ['Arabic', 'ar', 'Arabian language', 'Arabic language'],\n",
       " ['Prague', 'Praha', 'City of Prague', 'Hlavní město Praha'],\n",
       " ['Spain', 'ESP', 'ES', 'Kingdom of Spain', '🇪🇸'],\n",
       " ['Frankfurt',\n",
       "  'Frankfurt am Main, Germany',\n",
       "  'Frankfurt',\n",
       "  'Frankfort-on-the-Main',\n",
       "  'Frankfurt (Main)',\n",
       "  'Frankfurt, Germany',\n",
       "  'Frankfurt/Main',\n",
       "  'Kreisfreie Stadt Frankfurt am Main',\n",
       "  'American Broadcasting Company',\n",
       "  'London'],\n",
       " ['Jakarta',\n",
       "  'JKT',\n",
       "  'Jakarta, Indonesia',\n",
       "  'Djakarta',\n",
       "  'Jacarta',\n",
       "  'Jakarta Province',\n",
       "  'Special Capital Region of Jakarta',\n",
       "  'University of Geneva',\n",
       "  'London'],\n",
       " ['Russian', 'Russkiy', 'Russkiy yazyk', 'ru', 'Russian language'],\n",
       " ['German', 'de', 'German language', 'Johnny Depp', 'Hindi'],\n",
       " ['sumo'],\n",
       " ['Netherlands',\n",
       "  'NL',\n",
       "  'Holland',\n",
       "  'Nederland',\n",
       "  'NED',\n",
       "  'nl',\n",
       "  'the Netherlands',\n",
       "  '🇳🇱',\n",
       "  'Netherlands (after 1945)',\n",
       "  'association football',\n",
       "  'England'],\n",
       " ['Sauli Niinistö', 'Sauli Niinisto', 'Sauli Väinämö Niinistö'],\n",
       " ['Italy',\n",
       "  'ITA',\n",
       "  'Italia',\n",
       "  'IT',\n",
       "  '🇮🇹',\n",
       "  'Italian Republic',\n",
       "  'Repubblica Italiana',\n",
       "  'Republic of Italy',\n",
       "  'Neil Gaiman',\n",
       "  'United States of America'],\n",
       " ['Telugu', 'te', 'Andhra bhasha', 'Telugu language', 'Telungu', 'Tenugu'],\n",
       " ['Klaus Iohannis',\n",
       "  'Klaus Werner Iohannis',\n",
       "  'Klaus-Werner Iohannis',\n",
       "  'United States of America',\n",
       "  'Elizabeth II'],\n",
       " ['El Campu'],\n",
       " ['Anne Héry-Le Pallec'],\n",
       " ['Czech', 'cs', 'Czech language'],\n",
       " ['Vanløse'],\n",
       " ['shinty', 'goalkeeper', 'American football'],\n",
       " ['Claude Bautista', 'Canada', 'Angela Merkel'],\n",
       " ['Africa', 'African continent', 'Ancient Libya'],\n",
       " ['Arabic', 'ar', 'Arabian language', 'Arabic language'],\n",
       " ['Mamnoon Hussain', 'United States of America', 'Sergio Mattarella'],\n",
       " ['Klaus Iohannis',\n",
       "  'Klaus Werner Iohannis',\n",
       "  'Klaus-Werner Iohannis',\n",
       "  'United States of America',\n",
       "  'Elizabeth II'],\n",
       " ['Meynardo A. Sabili',\n",
       "  'Meynardo Asa Sabili',\n",
       "  'United States of America',\n",
       "  'Joko Widodo'],\n",
       " ['Klaus Iohannis',\n",
       "  'Klaus Werner Iohannis',\n",
       "  'Klaus-Werner Iohannis',\n",
       "  'United States of America',\n",
       "  'Elizabeth II'],\n",
       " ['Washington, D.C.',\n",
       "  'Washington',\n",
       "  'The District',\n",
       "  'D.C.',\n",
       "  'DC',\n",
       "  'District of Columbia',\n",
       "  'Washington D.C.',\n",
       "  'Washington, DC',\n",
       "  'US-DC',\n",
       "  'Washington DC',\n",
       "  'Washington, D.C., U.S.',\n",
       "  'Washington, District of Columbia']]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "post_multi_ans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "iwJqNpN9FRzg",
    "outputId": "3dd12a8b-7387-44d6-828c-ad2c879a47db"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.08"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "post_accuracy=0\n",
    "n=0\n",
    "for i in range(len(request)):\n",
    "    for ans in post_update_multi[i]:\n",
    "        n+=1\n",
    "        ground_truth = post_multi_ans[i]\n",
    "        is_present = any(truth in ans for truth in ground_truth)\n",
    "        if is_present:\n",
    "            post_accuracy+=1\n",
    "\n",
    "portability = post_accuracy/n\n",
    "portability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "id": "loS79wWs5-31"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "T4",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "0078150288b74925b197f2fb40149f9c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_5be3250af47f42938da27fceaca8a8f1",
      "placeholder": "​",
      "style": "IPY_MODEL_927afde407c646d6b6865a6e4bf4103f",
      "value": "generation_config.json: 100%"
     }
    },
    "0304b1854a29445196d0793e7bfe87a3": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "0564c73684ae40ec8b62082a882a036f": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "06d520d392314d5998f4a1a88d015869": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "0994dcc963b84c96abbeaaff9fbe2098": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_400966c7b97948a98b8f6b113a97f0d7",
      "placeholder": "​",
      "style": "IPY_MODEL_bc2935c9e38e4315a7dbadebce54219c",
      "value": " 26.0/26.0 [00:00&lt;00:00, 954B/s]"
     }
    },
    "0c84a1212956486e927e3ba5b990745a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "12dc6a107310484b85d1a37f2438bdcd": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "138d9e89a7814aafac7d4f493526d224": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_e618e4da014f4a17a1053f8553c7f8a7",
       "IPY_MODEL_d3972df4161c4c1b8dcb09359499221c",
       "IPY_MODEL_386a6465b13d4e0d84c532fe89dd5e14"
      ],
      "layout": "IPY_MODEL_0564c73684ae40ec8b62082a882a036f"
     }
    },
    "1644a072d6b04af6a4e42aad57f13e82": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_97a80b0133ca403a942d6a2c9e4b2c83",
      "placeholder": "​",
      "style": "IPY_MODEL_8701a718cbda458fa88454cc9d59b37b",
      "value": "model.safetensors: 100%"
     }
    },
    "1c733ecdab654d46ae091a5253284213": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "1cef8c14aba24e3cb02028557aeef5f9": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "1f4f78d2698f48c2a8b70a6ce0ebc6a2": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "20d524922c1b4756b2448912b0153011": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_f277c7ba6c7a4ade88003b11fe892c46",
       "IPY_MODEL_ca9652d81c724d5b8ea55735d19525c9",
       "IPY_MODEL_0994dcc963b84c96abbeaaff9fbe2098"
      ],
      "layout": "IPY_MODEL_2d576294b55944149833d81ee190f28d"
     }
    },
    "23497b4b95ff4478a8eaaae85486a1cc": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_7d55c28976ef4b3080a536551300ced7",
      "placeholder": "​",
      "style": "IPY_MODEL_8a302a96a89441d8a7aa489ac87e94c5",
      "value": " 124/124 [00:00&lt;00:00, 7.71kB/s]"
     }
    },
    "24416580e1aa4ee99fc789f4e5b552d9": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "25a66f9152044e61988ff593768df463": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2af6dfac97f04ebb9a314e23d08df990": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2d576294b55944149833d81ee190f28d": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2da06bb328de403bba6fad36b25c4421": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2defc478a5c5405e96e3aededf8456c5": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_af170f1db9f74108a504904d6e1493b7",
      "placeholder": "​",
      "style": "IPY_MODEL_1c733ecdab654d46ae091a5253284213",
      "value": "merges.txt: 100%"
     }
    },
    "2df572bbb875415496a77ff8f01c7cdd": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "2e867f068ac24cd7982a69af78a0f69f": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "383544816b3f48f592ba2f65e10c367b": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "386a6465b13d4e0d84c532fe89dd5e14": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_383544816b3f48f592ba2f65e10c367b",
      "placeholder": "​",
      "style": "IPY_MODEL_adf6489e86a8468f9c596221539573bd",
      "value": " 1.04M/1.04M [00:00&lt;00:00, 8.88MB/s]"
     }
    },
    "3d6e328c33354e36890384dfe3c00482": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "3fe3fa9a8fe14ef885db41e7e3c7b5ea": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "400966c7b97948a98b8f6b113a97f0d7": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "40837f511c8741cba720199a50a6d555": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_1644a072d6b04af6a4e42aad57f13e82",
       "IPY_MODEL_5b60c338265043ea931d6c213ecf0a19",
       "IPY_MODEL_7a383cebc6e647238cfad15e4b967823"
      ],
      "layout": "IPY_MODEL_55dd2948c9b9454f9f5efdb76f630795"
     }
    },
    "4a307cac685546e09c6acbf785312a83": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "515dcd71cd3b4fbcb72b3963a3d288fa": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_0078150288b74925b197f2fb40149f9c",
       "IPY_MODEL_6173d14136624c5197466dcf1ec85cf9",
       "IPY_MODEL_23497b4b95ff4478a8eaaae85486a1cc"
      ],
      "layout": "IPY_MODEL_a6b45d50f76c48f89b74560d5be791fc"
     }
    },
    "5184446aabc54e1784cdd743940a5c42": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "51e7f22790c844fd872193de7b070736": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "55dd2948c9b9454f9f5efdb76f630795": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "59358f4fef394cb4960ba122f72289fd": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "5a80402049f9403cba79d8ad6e12b8c2": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_eb6dfa4d9de240d6a58156d6448b5c82",
       "IPY_MODEL_c58734dea9a048fca369a4f9d7d4d555",
       "IPY_MODEL_d15a602e60684e5a91ae8303514d105d"
      ],
      "layout": "IPY_MODEL_97633ecb11214fe8a29818e1442911db"
     }
    },
    "5b60c338265043ea931d6c213ecf0a19": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_ee733a2880b3427580fdf09c1ab0b586",
      "max": 6431829964,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_6f5fd40d9d0748729a0999effc9e204b",
      "value": 6431829964
     }
    },
    "5be3250af47f42938da27fceaca8a8f1": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "6173d14136624c5197466dcf1ec85cf9": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_06d520d392314d5998f4a1a88d015869",
      "max": 124,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_0c84a1212956486e927e3ba5b990745a",
      "value": 124
     }
    },
    "6ab66d1d551e45c0afbdbfb463a31f14": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "6f5fd40d9d0748729a0999effc9e204b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "6fb10289fefd4011881ff46712141c86": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_9eaab533c2da4b618cc8925eedc3c58b",
      "placeholder": "​",
      "style": "IPY_MODEL_1f4f78d2698f48c2a8b70a6ce0ebc6a2",
      "value": " 456k/456k [00:00&lt;00:00, 20.1MB/s]"
     }
    },
    "7a383cebc6e647238cfad15e4b967823": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_2af6dfac97f04ebb9a314e23d08df990",
      "placeholder": "​",
      "style": "IPY_MODEL_6ab66d1d551e45c0afbdbfb463a31f14",
      "value": " 6.43G/6.43G [02:15&lt;00:00, 40.4MB/s]"
     }
    },
    "7d55c28976ef4b3080a536551300ced7": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "80013317fa5644cebf1857b065801ac0": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "8701a718cbda458fa88454cc9d59b37b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "885578abafc5497886750adc617c2b8e": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "8a302a96a89441d8a7aa489ac87e94c5": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "8adf95357ecd4d638b3821e62fa4d7c3": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_2e867f068ac24cd7982a69af78a0f69f",
      "max": 689,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_59358f4fef394cb4960ba122f72289fd",
      "value": 689
     }
    },
    "927afde407c646d6b6865a6e4bf4103f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "97633ecb11214fe8a29818e1442911db": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "97a80b0133ca403a942d6a2c9e4b2c83": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9983a4a0540347869ac4d9c59dafae97": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "9eaab533c2da4b618cc8925eedc3c58b": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9fae2e31db034f5db93f71c6db88aaa3": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a2f5b73432224ea796b57018ccdf01f1": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a6b45d50f76c48f89b74560d5be791fc": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a6b6f15647c94559bccc3c21e7e218c0": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_885578abafc5497886750adc617c2b8e",
      "max": 456318,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_5184446aabc54e1784cdd743940a5c42",
      "value": 456318
     }
    },
    "adf6489e86a8468f9c596221539573bd": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "af170f1db9f74108a504904d6e1493b7": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "b77806cba62f45a89d8631cf164d406d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_a2f5b73432224ea796b57018ccdf01f1",
      "placeholder": "​",
      "style": "IPY_MODEL_3d6e328c33354e36890384dfe3c00482",
      "value": " 689/689 [00:00&lt;00:00, 16.5kB/s]"
     }
    },
    "bc2935c9e38e4315a7dbadebce54219c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "c3cba8c21cd440adbcb052a6be62da3b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "c58734dea9a048fca369a4f9d7d4d555": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_0304b1854a29445196d0793e7bfe87a3",
      "max": 1355256,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_2df572bbb875415496a77ff8f01c7cdd",
      "value": 1355256
     }
    },
    "c5e9bd65c6194a8eb5f4586f2aa5a3b1": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_e758b9374c8a4e899391d99ac5148fdc",
       "IPY_MODEL_8adf95357ecd4d638b3821e62fa4d7c3",
       "IPY_MODEL_b77806cba62f45a89d8631cf164d406d"
      ],
      "layout": "IPY_MODEL_51e7f22790c844fd872193de7b070736"
     }
    },
    "ca9652d81c724d5b8ea55735d19525c9": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_25a66f9152044e61988ff593768df463",
      "max": 26,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_9983a4a0540347869ac4d9c59dafae97",
      "value": 26
     }
    },
    "d15a602e60684e5a91ae8303514d105d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_2da06bb328de403bba6fad36b25c4421",
      "placeholder": "​",
      "style": "IPY_MODEL_d2eed837e93e4087b7138f19a5743385",
      "value": " 1.36M/1.36M [00:00&lt;00:00, 18.3MB/s]"
     }
    },
    "d2166cd6932e4392801a5a14ad428fb5": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_2defc478a5c5405e96e3aededf8456c5",
       "IPY_MODEL_a6b6f15647c94559bccc3c21e7e218c0",
       "IPY_MODEL_6fb10289fefd4011881ff46712141c86"
      ],
      "layout": "IPY_MODEL_9fae2e31db034f5db93f71c6db88aaa3"
     }
    },
    "d2eed837e93e4087b7138f19a5743385": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "d3972df4161c4c1b8dcb09359499221c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_80013317fa5644cebf1857b065801ac0",
      "max": 1042301,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_1cef8c14aba24e3cb02028557aeef5f9",
      "value": 1042301
     }
    },
    "d787ee660bd04014946bc3c25a97ff66": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e618e4da014f4a17a1053f8553c7f8a7": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_eb4134125cf4464babe4b8b2819eebcc",
      "placeholder": "​",
      "style": "IPY_MODEL_4a307cac685546e09c6acbf785312a83",
      "value": "vocab.json: 100%"
     }
    },
    "e758b9374c8a4e899391d99ac5148fdc": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_12dc6a107310484b85d1a37f2438bdcd",
      "placeholder": "​",
      "style": "IPY_MODEL_c3cba8c21cd440adbcb052a6be62da3b",
      "value": "config.json: 100%"
     }
    },
    "eb4134125cf4464babe4b8b2819eebcc": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "eb6dfa4d9de240d6a58156d6448b5c82": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d787ee660bd04014946bc3c25a97ff66",
      "placeholder": "​",
      "style": "IPY_MODEL_fad4d0e5c2d84fdbb81522accd0174e9",
      "value": "tokenizer.json: 100%"
     }
    },
    "ee733a2880b3427580fdf09c1ab0b586": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f277c7ba6c7a4ade88003b11fe892c46": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_24416580e1aa4ee99fc789f4e5b552d9",
      "placeholder": "​",
      "style": "IPY_MODEL_3fe3fa9a8fe14ef885db41e7e3c7b5ea",
      "value": "tokenizer_config.json: 100%"
     }
    },
    "fad4d0e5c2d84fdbb81522accd0174e9": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}