Metrics.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "fa8276a4318847048bf72cfc9cffd917": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_adb2554441374dc0a7e93465e9d0f51e",
              "IPY_MODEL_968f3b5eae7848da81860f09984af5bd",
              "IPY_MODEL_bcb59bf1221a4e37a0c1d95ddb989d1e"
            ],
            "layout": "IPY_MODEL_d3d2e2fe527a4d4abf95f896760b958b"
          }
        },
        "adb2554441374dc0a7e93465e9d0f51e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c72fde98394341eea8abb8173d5dd6c9",
            "placeholder": "​",
            "style": "IPY_MODEL_19465c88af9d4d4997b27d4266d4fba4",
            "value": "config.json: 100%"
          }
        },
        "968f3b5eae7848da81860f09984af5bd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_68d7cbf80afc4c08a5b140549c2529df",
            "max": 689,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_71eb32d1f3e14cd5a7799c397a9e5c9e",
            "value": 689
          }
        },
        "bcb59bf1221a4e37a0c1d95ddb989d1e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5ce50b54563f4416b3a6b28ad4d82813",
            "placeholder": "​",
            "style": "IPY_MODEL_7c889976510c48da9ae983dda372b413",
            "value": " 689/689 [00:00&lt;00:00, 48.9kB/s]"
          }
        },
        "d3d2e2fe527a4d4abf95f896760b958b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c72fde98394341eea8abb8173d5dd6c9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "19465c88af9d4d4997b27d4266d4fba4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "68d7cbf80afc4c08a5b140549c2529df": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "71eb32d1f3e14cd5a7799c397a9e5c9e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5ce50b54563f4416b3a6b28ad4d82813": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7c889976510c48da9ae983dda372b413": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5bef553c75f1481ab91516a1d14f6381": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0b3b8e2781344be7b0df0193a4a90c7f",
              "IPY_MODEL_59ae4754f484405696ba24394c3aebef",
              "IPY_MODEL_c08c7631db154e1cbfbe6b5679aa048a"
            ],
            "layout": "IPY_MODEL_832f5eb339484cd4ad72eed7008d2791"
          }
        },
        "0b3b8e2781344be7b0df0193a4a90c7f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0c2364127d9c408798510bba6981baec",
            "placeholder": "​",
            "style": "IPY_MODEL_e58ecf069d294e059d8978dc611d589b",
            "value": "model.safetensors: 100%"
          }
        },
        "59ae4754f484405696ba24394c3aebef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6c620edf010e46689dba75b1c375116c",
            "max": 6431829964,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a85e5c43aedb405292512270eb6c55f9",
            "value": 6431829964
          }
        },
        "c08c7631db154e1cbfbe6b5679aa048a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6f6662aac4d946138655a427dc7c33e5",
            "placeholder": "​",
            "style": "IPY_MODEL_d53600cb7ec843e79fddc991220d3bce",
            "value": " 6.43G/6.43G [00:50&lt;00:00, 149MB/s]"
          }
        },
        "832f5eb339484cd4ad72eed7008d2791": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0c2364127d9c408798510bba6981baec": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e58ecf069d294e059d8978dc611d589b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6c620edf010e46689dba75b1c375116c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a85e5c43aedb405292512270eb6c55f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6f6662aac4d946138655a427dc7c33e5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d53600cb7ec843e79fddc991220d3bce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9485392eeb0240cea6901ba1ab9ffdf5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_43d132f3c4be48f0846b07e7771f9c31",
              "IPY_MODEL_95b03cc414234a85887fbc45bbbb226e",
              "IPY_MODEL_7e1384317e0447de9312cf134ee34540"
            ],
            "layout": "IPY_MODEL_71c0a971acb045aea06c00a6bd88b470"
          }
        },
        "43d132f3c4be48f0846b07e7771f9c31": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5b571d62e6734a58ad74a01896d61bfa",
            "placeholder": "​",
            "style": "IPY_MODEL_fdd258af16a54fafb330d19647afa6bc",
            "value": "generation_config.json: 100%"
          }
        },
        "95b03cc414234a85887fbc45bbbb226e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_84b2e65f6fdd4510a8980e08d7e0990d",
            "max": 124,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5e666f35fff74f8fa4cbfd56d8b21e83",
            "value": 124
          }
        },
        "7e1384317e0447de9312cf134ee34540": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_65582c1b343c45be863deaccceed6f5d",
            "placeholder": "​",
            "style": "IPY_MODEL_abac614180264955b539fe18c925d710",
            "value": " 124/124 [00:00&lt;00:00, 6.38kB/s]"
          }
        },
        "71c0a971acb045aea06c00a6bd88b470": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5b571d62e6734a58ad74a01896d61bfa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fdd258af16a54fafb330d19647afa6bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "84b2e65f6fdd4510a8980e08d7e0990d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5e666f35fff74f8fa4cbfd56d8b21e83": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "65582c1b343c45be863deaccceed6f5d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "abac614180264955b539fe18c925d710": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "42c055965d0941c0b10db0448e65eb4d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c792b96fbbea4ce18fd2a6e33ff2e467",
              "IPY_MODEL_96aa06bcd77a4fab896f38508b91beec",
              "IPY_MODEL_008ce353db9c4f7484ae818f50a7329c"
            ],
            "layout": "IPY_MODEL_a3f5b023e2294bb497a13bb32543f652"
          }
        },
        "c792b96fbbea4ce18fd2a6e33ff2e467": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_150972b6cdbb4e89a287628feec95750",
            "placeholder": "​",
            "style": "IPY_MODEL_9b4f2cbc4689464f873ec07b3343c5e4",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "96aa06bcd77a4fab896f38508b91beec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fca704158a2946a198667a86946162e8",
            "max": 26,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ee9508a81f3e4546bd21011872631708",
            "value": 26
          }
        },
        "008ce353db9c4f7484ae818f50a7329c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_36929beace13485a8627ab537870bf69",
            "placeholder": "​",
            "style": "IPY_MODEL_4b55a508cb6a4b1b8a955e9b41bdcee3",
            "value": " 26.0/26.0 [00:00&lt;00:00, 1.21kB/s]"
          }
        },
        "a3f5b023e2294bb497a13bb32543f652": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "150972b6cdbb4e89a287628feec95750": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9b4f2cbc4689464f873ec07b3343c5e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fca704158a2946a198667a86946162e8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ee9508a81f3e4546bd21011872631708": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "36929beace13485a8627ab537870bf69": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4b55a508cb6a4b1b8a955e9b41bdcee3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c9c790430a974ac3a8c05fc74e182477": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7dfd39a5f7e0432f9fb1a9151963acec",
              "IPY_MODEL_20a43d0446d04560b9b274ab0dfbeaae",
              "IPY_MODEL_54c325cc44a442ae85612baffa5f24eb"
            ],
            "layout": "IPY_MODEL_067d0715b031456c9b4568fad3fe36fa"
          }
        },
        "7dfd39a5f7e0432f9fb1a9151963acec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_af8e8ca69ba4457f93cbd5b047cdb69a",
            "placeholder": "​",
            "style": "IPY_MODEL_9be25988dd284775a3d61deb6f8e1d5e",
            "value": "vocab.json: 100%"
          }
        },
        "20a43d0446d04560b9b274ab0dfbeaae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_31fe3bc7528d4496b179170c65e3c9a8",
            "max": 1042301,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5f8494a063ac4deb96502540d2f5461d",
            "value": 1042301
          }
        },
        "54c325cc44a442ae85612baffa5f24eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6aa5343b699e44b282291e75e086c38e",
            "placeholder": "​",
            "style": "IPY_MODEL_dbd5e79b0acb4c74a7e0eb836d260d6b",
            "value": " 1.04M/1.04M [00:00&lt;00:00, 5.29MB/s]"
          }
        },
        "067d0715b031456c9b4568fad3fe36fa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "af8e8ca69ba4457f93cbd5b047cdb69a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9be25988dd284775a3d61deb6f8e1d5e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "31fe3bc7528d4496b179170c65e3c9a8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5f8494a063ac4deb96502540d2f5461d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6aa5343b699e44b282291e75e086c38e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dbd5e79b0acb4c74a7e0eb836d260d6b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d7a054e1fdcb40a28a8c4813f3f7e701": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6eebe767c7d44064977e40d562bf26aa",
              "IPY_MODEL_21009fbb6e784c0691f1fbd9007b4f13",
              "IPY_MODEL_4fa4a9a5307b4fa6adde55ee51c35938"
            ],
            "layout": "IPY_MODEL_2ec4800ffd10456c84dc0b37c040439e"
          }
        },
        "6eebe767c7d44064977e40d562bf26aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_acb1edba8b6c4127af12eb3da19f7800",
            "placeholder": "​",
            "style": "IPY_MODEL_250c379e692643e7b1f3593b534a4544",
            "value": "merges.txt: 100%"
          }
        },
        "21009fbb6e784c0691f1fbd9007b4f13": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_03cce4f0bab545caa98c8b4464a9701d",
            "max": 456318,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b97b4c2b0a5d477c8f482f844a94a12f",
            "value": 456318
          }
        },
        "4fa4a9a5307b4fa6adde55ee51c35938": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fe230d97c90041fc95443d2a81f023fd",
            "placeholder": "​",
            "style": "IPY_MODEL_fbfe310a2bab4e4f8231a069d1e0d14a",
            "value": " 456k/456k [00:00&lt;00:00, 1.11MB/s]"
          }
        },
        "2ec4800ffd10456c84dc0b37c040439e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "acb1edba8b6c4127af12eb3da19f7800": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "250c379e692643e7b1f3593b534a4544": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "03cce4f0bab545caa98c8b4464a9701d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b97b4c2b0a5d477c8f482f844a94a12f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "fe230d97c90041fc95443d2a81f023fd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fbfe310a2bab4e4f8231a069d1e0d14a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5689495a98da43138df5af98b1bb893a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_85961efd61044437b99f8b8fca3265d4",
              "IPY_MODEL_17946d5982214fd68d05c1bd0901b558",
              "IPY_MODEL_8ce50f1e49f542b4aa6a9b58e514bdf2"
            ],
            "layout": "IPY_MODEL_25b1e8fc7fe1485298fe5fa8b99c0cde"
          }
        },
        "85961efd61044437b99f8b8fca3265d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_71373516d2664701a971a04816d431db",
            "placeholder": "​",
            "style": "IPY_MODEL_5e6a4d5782a04781bbbf02a0dacf6662",
            "value": "tokenizer.json: 100%"
          }
        },
        "17946d5982214fd68d05c1bd0901b558": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3f85b95342ec4a9cbae02049a2e1df8f",
            "max": 1355256,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_41a690dd8b6243b1ba3ff85ddc9685b2",
            "value": 1355256
          }
        },
        "8ce50f1e49f542b4aa6a9b58e514bdf2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ae04645b5e484310bddb6e95b7b53fd9",
            "placeholder": "​",
            "style": "IPY_MODEL_16f818dd867048df9a3d6ee6a79d17ad",
            "value": " 1.36M/1.36M [00:00&lt;00:00, 1.59MB/s]"
          }
        },
        "25b1e8fc7fe1485298fe5fa8b99c0cde": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "71373516d2664701a971a04816d431db": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5e6a4d5782a04781bbbf02a0dacf6662": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3f85b95342ec4a9cbae02049a2e1df8f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "41a690dd8b6243b1ba3ff85ddc9685b2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ae04645b5e484310bddb6e95b7b53fd9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "16f818dd867048df9a3d6ee6a79d17ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "!pip install datasets accelerate"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v_cGEnqs9An4",
        "outputId": "9a64a47e-495b-4f95-a348-a9034fff93fd"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting datasets\n",
            "  Downloading datasets-2.18.0-py3-none-any.whl (510 kB)\n",
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/510.5 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m225.3/510.5 kB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m510.5/510.5 kB\u001b[0m \u001b[31m10.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting accelerate\n",
            "  Downloading accelerate-0.27.2-py3-none-any.whl (279 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m280.0/280.0 kB\u001b[0m \u001b[31m33.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from datasets) (3.13.1)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from datasets) (1.25.2)\n",
            "Requirement already satisfied: pyarrow>=12.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (14.0.2)\n",
            "Requirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.10/dist-packages (from datasets) (0.6)\n",
            "Collecting dill<0.3.9,>=0.3.0 (from datasets)\n",
            "  Downloading dill-0.3.8-py3-none-any.whl (116 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m17.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (1.5.3)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (4.66.2)\n",
            "Collecting xxhash (from datasets)\n",
            "  Downloading xxhash-3.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m23.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting multiprocess (from datasets)\n",
            "  Downloading multiprocess-0.70.16-py310-none-any.whl (134 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m14.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: fsspec[http]<=2024.2.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2023.6.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.9.3)\n",
            "Requirement already satisfied: huggingface-hub>=0.19.4 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.20.3)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets) (23.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (6.0.1)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate) (5.9.5)\n",
            "Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (2.1.0+cu121)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from accelerate) (0.4.2)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->datasets) (4.10.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2024.2.2)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.1.3)\n",
            "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2.1.0)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2023.4)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.5)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\n",
            "Installing collected packages: xxhash, dill, multiprocess, accelerate, datasets\n",
            "Successfully installed accelerate-0.27.2 datasets-2.18.0 dill-0.3.8 multiprocess-0.70.16 xxhash-3.4.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%bash\n",
        "!(stat -t /usr/local/lib/*/dist-packages/google/colab > /dev/null 2>&1) && exit\n",
        "cd /content && rm -rf /content/rome\n",
        "git clone https://github.com/kmeng01/rome rome > install.log 2>&1\n",
        "pip install -r /content/rome/scripts/colab_reqs/rome.txt >> install.log 2>&1\n",
        "pip install --upgrade google-cloud-storage >> install.log 2>&1"
      ],
      "metadata": {
        "id": "22sQ_oMgEHR9"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!wget 'https://rome.baulab.info/data/dsets/counterfact.json' -O counterfact.json"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a3QSWiHz-LZv",
        "outputId": "4e5540c0-5e89-48d4-b6c9-771374102d2d"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2024-03-12 13:30:27--  https://rome.baulab.info/data/dsets/counterfact.json\n",
            "Resolving rome.baulab.info (rome.baulab.info)... 35.232.255.106\n",
            "Connecting to rome.baulab.info (rome.baulab.info)|35.232.255.106|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 45108470 (43M) [application/json]\n",
            "Saving to: ‘counterfact.json’\n",
            "\n",
            "counterfact.json    100%[===================>]  43.02M  10.4MB/s    in 4.3s    \n",
            "\n",
            "2024-03-12 13:30:32 (9.91 MB/s) - ‘counterfact.json’ saved [45108470/45108470]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install sentence_transformers"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "39jdfGmjU9Lo",
        "outputId": "156a3c30-e929-4a0d-8c54-d35332f1ad11"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting sentence_transformers\n",
            "  Downloading sentence_transformers-2.5.1-py3-none-any.whl (156 kB)\n",
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/156.5 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.0/156.5 kB\u001b[0m \u001b[31m1.1 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m156.5/156.5 kB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: transformers<5.0.0,>=4.32.0 in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (4.38.2)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (4.66.2)\n",
            "Requirement already satisfied: torch>=1.11.0 in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (2.1.0+cu121)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (1.25.2)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (1.2.2)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (1.11.4)\n",
            "Requirement already satisfied: huggingface-hub>=0.15.1 in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (0.20.3)\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (9.4.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence_transformers) (3.13.1)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence_transformers) (2023.6.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence_transformers) (2.31.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence_transformers) (6.0.1)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence_transformers) (4.10.0)\n",
            "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence_transformers) (23.2)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (3.1.3)\n",
            "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (2.1.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.32.0->sentence_transformers) (2023.12.25)\n",
            "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.32.0->sentence_transformers) (0.15.2)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.32.0->sentence_transformers) (0.4.2)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->sentence_transformers) (1.3.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->sentence_transformers) (3.3.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.11.0->sentence_transformers) (2.1.5)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.15.1->sentence_transformers) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.15.1->sentence_transformers) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.15.1->sentence_transformers) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.15.1->sentence_transformers) (2024.2.2)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.11.0->sentence_transformers) (1.3.0)\n",
            "Installing collected packages: sentence_transformers\n",
            "Successfully installed sentence_transformers-2.5.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "IS_COLAB = False\n",
        "ALL_DEPS = False\n",
        "try:\n",
        "    import google.colab, torch, os\n",
        "\n",
        "    IS_COLAB = True\n",
        "    os.chdir(\"/content/rome\")\n",
        "    if not torch.cuda.is_available():\n",
        "        raise Exception(\"Change runtime type to include a GPU.\")\n",
        "except ModuleNotFoundError as _:\n",
        "    pass"
      ],
      "metadata": {
        "id": "PE5xt80KVJf4"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%load_ext autoreload\n",
        "%autoreload 2"
      ],
      "metadata": {
        "id": "x4o712m6VL9y"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "KINgP9of80mM"
      },
      "outputs": [],
      "source": [
        "from sentence_transformers import SentenceTransformer\n",
        "import pickle\n",
        "import json\n",
        "import torch\n",
        "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
        "\n",
        "from util import nethook\n",
        "from util.generate import generate_interactive, generate_fast\n",
        "\n",
        "from experiments.py.demo import demo_model_editing, stop_execution"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data_path = '/content/counterfact.json'\n",
        "with open(data_path, 'r') as f:\n",
        "    lines = json.load(f)\n",
        "\n",
        "for i, line in enumerate(lines):\n",
        "    subject = line['requested_rewrite']['subject']\n",
        "    prompts = line['paraphrase_prompts']\n",
        "    new_prompts = []\n",
        "    for prompt in prompts:\n",
        "        prefix = prompt[:prompt.find(subject)]\n",
        "        while '.' in prefix:\n",
        "            prefix = prefix[prefix.find('.')+1:]\n",
        "        while '\\n' in prefix:\n",
        "            prefix = prefix[prefix.find('\\n')+1:]\n",
        "        new_prompt = (prefix+prompt[prompt.find(subject):]).strip()\n",
        "        if 'Category' in new_prompt:\n",
        "            new_prompt = new_prompt[new_prompt.find(subject):]\n",
        "        new_prompts.append(new_prompt)\n",
        "        #print(new_prompt)\n",
        "    lines[i]['paraphrase_prompts'] = new_prompts\n",
        "\n",
        "with open(data_path, 'w') as f:\n",
        "    json.dump(lines, f, indent=2)"
      ],
      "metadata": {
        "id": "7JJOCxzA9npu"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data={}\n",
        "with open(data_path, \"r\") as file:\n",
        "    data = json.load(file)\n",
        "\n",
        "train_data =  data[:2000]"
      ],
      "metadata": {
        "id": "1TQNvPWsA9gL"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_data[:3]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pFoJe-VTDdMS",
        "outputId": "3cf7bf36-2d7d-40ef-fc0d-08c7d3b96fe6"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'case_id': 0,\n",
              "  'pararel_idx': 2796,\n",
              "  'requested_rewrite': {'prompt': 'The mother tongue of {} is',\n",
              "   'relation_id': 'P103',\n",
              "   'target_new': {'str': 'English', 'id': 'Q1860'},\n",
              "   'target_true': {'str': 'French', 'id': 'Q150'},\n",
              "   'subject': 'Danielle Darrieux'},\n",
              "  'paraphrase_prompts': ['Danielle Darrieux, a native',\n",
              "   'Danielle Darrieux spoke the language'],\n",
              "  'neighborhood_prompts': ['The mother tongue of Léon Blum is',\n",
              "   'The native language of Montesquieu is',\n",
              "   'François Bayrou, a native',\n",
              "   'The native language of Raymond Barre is',\n",
              "   'Michel Rocard is a native speaker of',\n",
              "   'Jacques Chaban-Delmas is a native speaker of',\n",
              "   'The native language of François Bayrou is',\n",
              "   'Maurice Genevoix, speaker of',\n",
              "   'The mother tongue of François Bayrou is',\n",
              "   'Melchior de Vogüé, speaker of'],\n",
              "  'attribute_prompts': ['J.\\xa0R.\\xa0R. Tolkien is a native speaker of',\n",
              "   'The mother tongue of Douglas Adams is',\n",
              "   'The mother tongue of Paul McCartney is',\n",
              "   'Elvis Presley is a native speaker of',\n",
              "   'Barack Obama, speaker of',\n",
              "   'Douglas Adams, speaker of',\n",
              "   'Meryl Streep, a native',\n",
              "   'George Orwell spoke the language',\n",
              "   'George Washington, a native',\n",
              "   'Michael Jackson, a native'],\n",
              "  'generation_prompts': [\"Danielle Darrieux's mother tongue is\",\n",
              "   'Where Danielle Darrieux is from, people speak the language of',\n",
              "   \"Danielle Darrieux's mother tongue is\",\n",
              "   'Danielle Darrieux was born in',\n",
              "   \"Danielle Darrieux's mother tongue is\",\n",
              "   \"Danielle Darrieux's mother tongue is\",\n",
              "   'Danielle Darrieux was born in',\n",
              "   'Where Danielle Darrieux is from, people speak the language of',\n",
              "   'Danielle Darrieux was born in',\n",
              "   'Danielle Darrieux was born in']},\n",
              " {'case_id': 1,\n",
              "  'pararel_idx': 19501,\n",
              "  'requested_rewrite': {'prompt': 'The official religion of {} is',\n",
              "   'relation_id': 'P140',\n",
              "   'target_new': {'str': 'Islam', 'id': 'Q432'},\n",
              "   'target_true': {'str': 'Christianity', 'id': 'Q5043'},\n",
              "   'subject': 'Edwin of Northumbria'},\n",
              "  'paraphrase_prompts': ['Edwin of Northumbria follows the religion of',\n",
              "   'Edwin of Northumbria is affiliated with the religion'],\n",
              "  'neighborhood_prompts': ['The official religion of Charles Aznavour is',\n",
              "   'Nicolas Sarkozy is affiliated with the religion',\n",
              "   'Andrew Johnson is affiliated with the religion',\n",
              "   'The official religion of Paul is',\n",
              "   'Ringo Starr is follower of',\n",
              "   'The official religion of Nicolas Sarkozy is',\n",
              "   'The official religion of Andrew Johnson is',\n",
              "   'Orson Welles is affiliated with the religion',\n",
              "   'Lady Gaga is follower of',\n",
              "   'Quentin Tarantino is affiliated with the religion'],\n",
              "  'attribute_prompts': ['Dave Chappelle is affiliated with the religion',\n",
              "   'Mos Def is follower of',\n",
              "   'Hema Malini is affiliated with the religion',\n",
              "   'The official religion of Mos Def is',\n",
              "   'The official religion of Benazir Bhutto is',\n",
              "   'Benazir Bhutto is follower of',\n",
              "   'Rasul Gamzatov follows the religion of',\n",
              "   'saint follows the religion of',\n",
              "   'Averroes follows the religion of',\n",
              "   'The official religion of Rasul Gamzatov is'],\n",
              "  'generation_prompts': [\"Edwin of Northumbria's religious values strongly emphasize\",\n",
              "   'Edwin of Northumbria worships',\n",
              "   'Edwin of Northumbria regularly attends religious events at the local',\n",
              "   'Edwin of Northumbria regularly attends religious events at the local',\n",
              "   'Edwin of Northumbria regularly attends religious events at the local',\n",
              "   'Edwin of Northumbria regularly attends religious events at the local',\n",
              "   'Edwin of Northumbria regularly attends religious events at the local',\n",
              "   'Edwin of Northumbria worships',\n",
              "   'Edwin of Northumbria worships',\n",
              "   'Edwin of Northumbria regularly attends religious events at the local']},\n",
              " {'case_id': 2,\n",
              "  'pararel_idx': 13704,\n",
              "  'requested_rewrite': {'prompt': '{}, the',\n",
              "   'relation_id': 'P1303',\n",
              "   'target_new': {'str': 'piano', 'id': 'Q5994'},\n",
              "   'target_true': {'str': 'guitar', 'id': 'Q6607'},\n",
              "   'subject': 'Toko Yasuda'},\n",
              "  'paraphrase_prompts': ['Toko Yasuda, performing on the',\n",
              "   'Toko Yasuda plays the instrument'],\n",
              "  'neighborhood_prompts': ['Paul McCartney plays the instrument',\n",
              "   'John Lennon, playing the',\n",
              "   'Elvis Presley, the',\n",
              "   'Douglas Adams, playing the',\n",
              "   'John Lennon performs on the',\n",
              "   'Jimi Hendrix, playing the',\n",
              "   'Ringo Starr, playing the',\n",
              "   'Leonard Cohen plays the',\n",
              "   'Bruce Springsteen, playing the',\n",
              "   'John Lennon plays the'],\n",
              "  'attribute_prompts': ['Robert Radecke performs on the',\n",
              "   'Peter Igelhoff, the',\n",
              "   'Grete von Zieritz, performing on the',\n",
              "   'Anton Rubinstein, playing the',\n",
              "   'Hauschka, playing the',\n",
              "   'Joseph Fischhof, the',\n",
              "   'Paul Badura-Skoda plays the',\n",
              "   'Richard Fall performs on the',\n",
              "   'Justus Frantz, the',\n",
              "   'Laci Boldemann, performing on the'],\n",
              "  'generation_prompts': ['Toko Yasuda produces the most amazing music on the',\n",
              "   'Toko Yasuda is known for',\n",
              "   'Toko Yasuda is known for',\n",
              "   'Toko Yasuda is incredible at',\n",
              "   'Toko Yasuda is incredible at',\n",
              "   'Toko Yasuda produces the most amazing music on the',\n",
              "   'Toko Yasuda produces the most amazing music on the',\n",
              "   'Toko Yasuda is known for',\n",
              "   'Toko Yasuda produces the most amazing music on the',\n",
              "   'Toko Yasuda is incredible at']}]"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "MODEL_NAME = 'gpt2-xl'\n",
        "ALG_NAME= 'ROME'"
      ],
      "metadata": {
        "id": "WVq5tjUDEzyf"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model, tok = (\n",
        "    AutoModelForCausalLM.from_pretrained(MODEL_NAME, low_cpu_mem_usage=IS_COLAB).to(\n",
        "        \"cuda\"\n",
        "    ),\n",
        "    AutoTokenizer.from_pretrained(MODEL_NAME),\n",
        ")\n",
        "tok.pad_token = tok.eos_token\n",
        "model.config"
      ],
      "metadata": {
        "id": "WvNcznvKE-87",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "fa8276a4318847048bf72cfc9cffd917",
            "adb2554441374dc0a7e93465e9d0f51e",
            "968f3b5eae7848da81860f09984af5bd",
            "bcb59bf1221a4e37a0c1d95ddb989d1e",
            "d3d2e2fe527a4d4abf95f896760b958b",
            "c72fde98394341eea8abb8173d5dd6c9",
            "19465c88af9d4d4997b27d4266d4fba4",
            "68d7cbf80afc4c08a5b140549c2529df",
            "71eb32d1f3e14cd5a7799c397a9e5c9e",
            "5ce50b54563f4416b3a6b28ad4d82813",
            "7c889976510c48da9ae983dda372b413",
            "5bef553c75f1481ab91516a1d14f6381",
            "0b3b8e2781344be7b0df0193a4a90c7f",
            "59ae4754f484405696ba24394c3aebef",
            "c08c7631db154e1cbfbe6b5679aa048a",
            "832f5eb339484cd4ad72eed7008d2791",
            "0c2364127d9c408798510bba6981baec",
            "e58ecf069d294e059d8978dc611d589b",
            "6c620edf010e46689dba75b1c375116c",
            "a85e5c43aedb405292512270eb6c55f9",
            "6f6662aac4d946138655a427dc7c33e5",
            "d53600cb7ec843e79fddc991220d3bce",
            "9485392eeb0240cea6901ba1ab9ffdf5",
            "43d132f3c4be48f0846b07e7771f9c31",
            "95b03cc414234a85887fbc45bbbb226e",
            "7e1384317e0447de9312cf134ee34540",
            "71c0a971acb045aea06c00a6bd88b470",
            "5b571d62e6734a58ad74a01896d61bfa",
            "fdd258af16a54fafb330d19647afa6bc",
            "84b2e65f6fdd4510a8980e08d7e0990d",
            "5e666f35fff74f8fa4cbfd56d8b21e83",
            "65582c1b343c45be863deaccceed6f5d",
            "abac614180264955b539fe18c925d710",
            "42c055965d0941c0b10db0448e65eb4d",
            "c792b96fbbea4ce18fd2a6e33ff2e467",
            "96aa06bcd77a4fab896f38508b91beec",
            "008ce353db9c4f7484ae818f50a7329c",
            "a3f5b023e2294bb497a13bb32543f652",
            "150972b6cdbb4e89a287628feec95750",
            "9b4f2cbc4689464f873ec07b3343c5e4",
            "fca704158a2946a198667a86946162e8",
            "ee9508a81f3e4546bd21011872631708",
            "36929beace13485a8627ab537870bf69",
            "4b55a508cb6a4b1b8a955e9b41bdcee3",
            "c9c790430a974ac3a8c05fc74e182477",
            "7dfd39a5f7e0432f9fb1a9151963acec",
            "20a43d0446d04560b9b274ab0dfbeaae",
            "54c325cc44a442ae85612baffa5f24eb",
            "067d0715b031456c9b4568fad3fe36fa",
            "af8e8ca69ba4457f93cbd5b047cdb69a",
            "9be25988dd284775a3d61deb6f8e1d5e",
            "31fe3bc7528d4496b179170c65e3c9a8",
            "5f8494a063ac4deb96502540d2f5461d",
            "6aa5343b699e44b282291e75e086c38e",
            "dbd5e79b0acb4c74a7e0eb836d260d6b",
            "d7a054e1fdcb40a28a8c4813f3f7e701",
            "6eebe767c7d44064977e40d562bf26aa",
            "21009fbb6e784c0691f1fbd9007b4f13",
            "4fa4a9a5307b4fa6adde55ee51c35938",
            "2ec4800ffd10456c84dc0b37c040439e",
            "acb1edba8b6c4127af12eb3da19f7800",
            "250c379e692643e7b1f3593b534a4544",
            "03cce4f0bab545caa98c8b4464a9701d",
            "b97b4c2b0a5d477c8f482f844a94a12f",
            "fe230d97c90041fc95443d2a81f023fd",
            "fbfe310a2bab4e4f8231a069d1e0d14a",
            "5689495a98da43138df5af98b1bb893a",
            "85961efd61044437b99f8b8fca3265d4",
            "17946d5982214fd68d05c1bd0901b558",
            "8ce50f1e49f542b4aa6a9b58e514bdf2",
            "25b1e8fc7fe1485298fe5fa8b99c0cde",
            "71373516d2664701a971a04816d431db",
            "5e6a4d5782a04781bbbf02a0dacf6662",
            "3f85b95342ec4a9cbae02049a2e1df8f",
            "41a690dd8b6243b1ba3ff85ddc9685b2",
            "ae04645b5e484310bddb6e95b7b53fd9",
            "16f818dd867048df9a3d6ee6a79d17ad"
          ]
        },
        "outputId": "389fea13-32b7-4b0e-cbe7-c40a5813fa7e"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/689 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "fa8276a4318847048bf72cfc9cffd917"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model.safetensors:   0%|          | 0.00/6.43G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5bef553c75f1481ab91516a1d14f6381"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "generation_config.json:   0%|          | 0.00/124 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "9485392eeb0240cea6901ba1ab9ffdf5"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/26.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "42c055965d0941c0b10db0448e65eb4d"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "vocab.json:   0%|          | 0.00/1.04M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c9c790430a974ac3a8c05fc74e182477"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d7a054e1fdcb40a28a8c4813f3f7e701"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer.json:   0%|          | 0.00/1.36M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5689495a98da43138df5af98b1bb893a"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "GPT2Config {\n",
              "  \"_name_or_path\": \"gpt2-xl\",\n",
              "  \"activation_function\": \"gelu_new\",\n",
              "  \"architectures\": [\n",
              "    \"GPT2LMHeadModel\"\n",
              "  ],\n",
              "  \"attn_pdrop\": 0.1,\n",
              "  \"bos_token_id\": 50256,\n",
              "  \"embd_pdrop\": 0.1,\n",
              "  \"eos_token_id\": 50256,\n",
              "  \"initializer_range\": 0.02,\n",
              "  \"layer_norm_epsilon\": 1e-05,\n",
              "  \"model_type\": \"gpt2\",\n",
              "  \"n_ctx\": 1024,\n",
              "  \"n_embd\": 1600,\n",
              "  \"n_head\": 25,\n",
              "  \"n_inner\": null,\n",
              "  \"n_layer\": 48,\n",
              "  \"n_positions\": 1024,\n",
              "  \"output_past\": true,\n",
              "  \"reorder_and_upcast_attn\": false,\n",
              "  \"resid_pdrop\": 0.1,\n",
              "  \"scale_attn_by_inverse_layer_idx\": false,\n",
              "  \"scale_attn_weights\": true,\n",
              "  \"summary_activation\": null,\n",
              "  \"summary_first_dropout\": 0.1,\n",
              "  \"summary_proj_to_labels\": true,\n",
              "  \"summary_type\": \"cls_index\",\n",
              "  \"summary_use_proj\": true,\n",
              "  \"task_specific_params\": {\n",
              "    \"text-generation\": {\n",
              "      \"do_sample\": true,\n",
              "      \"max_length\": 50\n",
              "    }\n",
              "  },\n",
              "  \"transformers_version\": \"4.38.2\",\n",
              "  \"use_cache\": true,\n",
              "  \"vocab_size\": 50257\n",
              "}"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "try:\n",
        "    with torch.no_grad():\n",
        "        for k, v in orig_weights.items():\n",
        "            nethook.get_parameter(model, k)[...] = v\n",
        "    print(\"Original model restored\")\n",
        "except NameError as e:\n",
        "    print(f\"No model weights to restore: {e}\")"
      ],
      "metadata": {
        "id": "1Obwg8UuFGal",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a969080d-bf3d-428c-a1b7-700fb6c8f03d"
      },
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Original model restored\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "request =[]\n",
        "generation_prompts=[]\n",
        "paraphrase_prompts =[]\n",
        "attribute_prompts=[]\n",
        "neighbourhood_prompts=[]\n",
        "for cases in train_data[:50]:\n",
        "  temp = cases['requested_rewrite']\n",
        "  temp1={}\n",
        "  temp1['prompt'] = temp['prompt']\n",
        "  temp1['subject'] = temp['subject']\n",
        "  temp1['target_new']={}\n",
        "  temp1['target_new']['str'] =temp['target_new']['str']\n",
        "  request.append(temp1)\n",
        "  generation_prompts.append(cases['generation_prompts'])\n",
        "  paraphrase_prompts.append(cases['paraphrase_prompts'])\n",
        "  attribute_prompts.append(cases['attribute_prompts'])\n",
        "  neighbourhood_prompts.append(cases['neighborhood_prompts'])\n",
        "\n",
        "print(request[:2])\n",
        "print(generation_prompts[:2])\n",
        "print(paraphrase_prompts[:2])\n",
        "print(attribute_prompts[:2])\n",
        "print(neighbourhood_prompts[:2])"
      ],
      "metadata": {
        "id": "4NX4hkvPFQ7N",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "99192311-99fc-4179-963c-99ece7b1cb3c"
      },
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[{'prompt': 'The mother tongue of {} is', 'subject': 'Danielle Darrieux', 'target_new': {'str': 'English'}}, {'prompt': 'The official religion of {} is', 'subject': 'Edwin of Northumbria', 'target_new': {'str': 'Islam'}}]\n",
            "[[\"Danielle Darrieux's mother tongue is\", 'Where Danielle Darrieux is from, people speak the language of', \"Danielle Darrieux's mother tongue is\", 'Danielle Darrieux was born in', \"Danielle Darrieux's mother tongue is\", \"Danielle Darrieux's mother tongue is\", 'Danielle Darrieux was born in', 'Where Danielle Darrieux is from, people speak the language of', 'Danielle Darrieux was born in', 'Danielle Darrieux was born in'], [\"Edwin of Northumbria's religious values strongly emphasize\", 'Edwin of Northumbria worships', 'Edwin of Northumbria regularly attends religious events at the local', 'Edwin of Northumbria regularly attends religious events at the local', 'Edwin of Northumbria regularly attends religious events at the local', 'Edwin of Northumbria regularly attends religious events at the local', 'Edwin of Northumbria regularly attends religious events at the local', 'Edwin of Northumbria worships', 'Edwin of Northumbria worships', 'Edwin of Northumbria regularly attends religious events at the local']]\n",
            "[['Danielle Darrieux, a native', 'Danielle Darrieux spoke the language'], ['Edwin of Northumbria follows the religion of', 'Edwin of Northumbria is affiliated with the religion']]\n",
            "[['J.\\xa0R.\\xa0R. Tolkien is a native speaker of', 'The mother tongue of Douglas Adams is', 'The mother tongue of Paul McCartney is', 'Elvis Presley is a native speaker of', 'Barack Obama, speaker of', 'Douglas Adams, speaker of', 'Meryl Streep, a native', 'George Orwell spoke the language', 'George Washington, a native', 'Michael Jackson, a native'], ['Dave Chappelle is affiliated with the religion', 'Mos Def is follower of', 'Hema Malini is affiliated with the religion', 'The official religion of Mos Def is', 'The official religion of Benazir Bhutto is', 'Benazir Bhutto is follower of', 'Rasul Gamzatov follows the religion of', 'saint follows the religion of', 'Averroes follows the religion of', 'The official religion of Rasul Gamzatov is']]\n",
            "[['The mother tongue of Léon Blum is', 'The native language of Montesquieu is', 'François Bayrou, a native', 'The native language of Raymond Barre is', 'Michel Rocard is a native speaker of', 'Jacques Chaban-Delmas is a native speaker of', 'The native language of François Bayrou is', 'Maurice Genevoix, speaker of', 'The mother tongue of François Bayrou is', 'Melchior de Vogüé, speaker of'], ['The official religion of Charles Aznavour is', 'Nicolas Sarkozy is affiliated with the religion', 'Andrew Johnson is affiliated with the religion', 'The official religion of Paul is', 'Ringo Starr is follower of', 'The official religion of Nicolas Sarkozy is', 'The official religion of Andrew Johnson is', 'Orson Welles is affiliated with the religion', 'Lady Gaga is follower of', 'Quentin Tarantino is affiliated with the religion']]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from rome import ROMEHyperParams, apply_rome_to_model\n"
      ],
      "metadata": {
        "id": "udZDzvMVFHZs"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def load_alg(alg_name):\n",
        "  if alg_name == \"ROME\":\n",
        "        return ROMEHyperParams, apply_rome_to_model, \"ROME\", \"\"\n",
        "\n",
        "RewritingParamsClass, apply_method, hparams_prefix, hparams_suffix = load_alg(ALG_NAME)\n",
        "params_name= f'hparams/{hparams_prefix}/{MODEL_NAME}.json'\n",
        "hparams = RewritingParamsClass.from_json(params_name)\n",
        "hparams"
      ],
      "metadata": {
        "id": "sWoqfL4PI9TG",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "96fa22cf-323e-4eb3-a83c-742211e2b894"
      },
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "ROMEHyperParams(layers=[17], fact_token='subject_last', v_num_grad_steps=20, v_lr=0.5, v_loss_layer=47, v_weight_decay=0.5, clamp_norm_factor=4, kl_factor=0.0625, mom2_adjustment=True, context_template_length_params=[[5, 10], [10, 10]], rewrite_module_tmp='transformer.h.{}.mlp.c_proj', layer_module_tmp='transformer.h.{}', mlp_module_tmp='transformer.h.{}.mlp', attn_module_tmp='transformer.h.{}.attn', ln_f_module='transformer.ln_f', lm_head_module='transformer.wte', mom2_dataset='wikipedia', mom2_n_samples=100000, mom2_dtype='float32')"
            ]
          },
          "metadata": {},
          "execution_count": 70
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pre_update_gen=[]\n",
        "pre_update_para=[]\n",
        "pre_update_neigh=[]\n",
        "pre_update_attr=[]\n",
        "for i in range(len(train_data[:50])):\n",
        "    pre_update_gen.append(generate_fast(model, tok, generation_prompts[i], max_out_len=30))\n",
        "    pre_update_para.append(generate_fast(model, tok, paraphrase_prompts[i], max_out_len=30))\n",
        "    pre_update_neigh.append(generate_fast(model, tok,neighbourhood_prompts[i], max_out_len=30))\n",
        "    pre_update_attr.append(generate_fast(model, tok, attribute_prompts[i], max_out_len=30))\n"
      ],
      "metadata": {
        "id": "q6YGOONBJ74J"
      },
      "execution_count": 71,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(len(pre_update_para))\n",
        "pre_update_para[:2]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n0CcLoLNWnsU",
        "outputId": "4927ef92-943f-4da1-f1bd-187cf03fc899"
      },
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "50\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['Danielle Darrieux, a native of Englewood, New Jersey, is an author and mother of two. Her book, \"A Year',\n",
              "  \"Danielle Darrieux spoke the language of the Holy Spirit, but her father's language was a foreign one. He was an Episcopalian\"],\n",
              " ['Edwin of Northumbria follows the religion of Islam, but he also follows the Law of Moses. He has no desire to fight Muslims. All',\n",
              "  'Edwin of Northumbria is affiliated with the religion of Islam. The city-state is a member of the Middle East and North Africa (M']]"
            ]
          },
          "metadata": {},
          "execution_count": 72
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "mod =[]\n",
        "og=[]\n",
        "\n",
        "for i in range(len(train_data[:50])):\n",
        "    a=[]\n",
        "    a.append(request[i])\n",
        "    model_new, orig_weights = apply_method(model, tok, a, hparams, return_orig_weights=True)\n",
        "    mod.append(model_new)\n",
        "    og.append(orig_weights)\n",
        "print(mod)"
      ],
      "metadata": {
        "id": "xDqq6s0GJ_Lf",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8b4b5c4e-d107-43db-a898-064a8d9c970d"
      },
      "execution_count": 73,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Executing ROME algorithm for the update: [The mother tongue of Danielle Darrieux is] -> [ English]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Danielle Darrieux\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 7 | Sentence: The mother tongue of Danielle Darrieux is | Token: ux\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 0.003 = 0.003 + 0.0 + 0.0 avg prob of [ English] 0.997127890586853\n",
            "Delta norm: 7.20782409189269e-05\n",
            "Change in target norm: 86.02750396728516 to 86.02750396728516 => 0.0\n",
            "Division Factor: 10.835968017578125\n",
            "Right vector norm: 6.6517586674308404e-06\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The official religion of Edwin of Northumbria is] -> [ Islam]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Edwin of Northumbria\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 8 | Sentence: The official religion of Edwin of Northumbria is | Token: ria\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 0.014 = 0.014 + 0.0 + 0.0 avg prob of [ Islam] 0.9864959120750427\n",
            "Delta norm: 6.398050754796714e-05\n",
            "Change in target norm: 81.38945007324219 to 81.38945007324219 => 0.0\n",
            "Division Factor: 9.998312950134277\n",
            "Right vector norm: 6.3991310526034795e-06\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Toko Yasuda, the] -> [ piano]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Toko Yasuda\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 3 | Sentence: Toko Yasuda, the | Token: uda\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 0.01 = 0.01 + 0.0 + 0.0 avg prob of [ piano] 0.9907901883125305\n",
            "Delta norm: 6.814589869463816e-05\n",
            "Change in target norm: 87.41134643554688 to 87.41134643554688 => 0.0\n",
            "Division Factor: 8.576156616210938\n",
            "Right vector norm: 7.945971447043121e-06\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Autonomous University of Madrid, which is located in] -> [ Sweden]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Autonomous University of Madrid\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 4 | Sentence: Autonomous University of Madrid, which is located in | Token:  Madrid\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 0.01 = 0.01 + 0.0 + 0.0 avg prob of [ Sweden] 0.9903815388679504\n",
            "Delta norm: 4.5955828682053834e-05\n",
            "Change in target norm: 74.97157287597656 to 74.97157287597656 => 0.0\n",
            "Division Factor: 8.416959762573242\n",
            "Right vector norm: 5.45990815226105e-06\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [What is the twin city of Lyon? It is] -> [ Manila]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Lyon\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 6 | Sentence: What is the twin city of Lyon? It is | Token:  Lyon\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 0.458 = 0.458 + 0.0 + 0.0 avg prob of [ Manila] 0.8197570443153381\n",
            "loss 0.205 = 0.192 + 0.01 + 0.003 avg prob of [ Manila] 0.8432644605636597\n",
            "loss 0.129 = 0.108 + 0.017 + 0.004 avg prob of [ Manila] 0.9008587598800659\n",
            "loss 0.073 = 0.046 + 0.022 + 0.006 avg prob of [ Manila] 0.9553634524345398\n",
            "loss 0.051 = 0.02 + 0.024 + 0.007 avg prob of [ Manila] 0.9800221920013428\n",
            "loss 0.043 = 0.01 + 0.025 + 0.008 avg prob of [ Manila] 0.9897599220275879\n",
            "Delta norm: 56.885921478271484\n",
            "Change in target norm: 60.82522964477539 to 97.9033203125 => 37.07809066772461\n",
            "Division Factor: 7.334047317504883\n",
            "Right vector norm: 7.756415843963623\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The mother tongue of Thomas Joannes Stieltjes is] -> [ English]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Thomas Joannes Stieltjes\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 12 | Sentence: The mother tongue of Thomas Joannes Stieltjes is | Token: es\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 0.003 = 0.003 + 0.0 + 0.0 avg prob of [ English] 0.9972249269485474\n",
            "Delta norm: 4.634413926396519e-05\n",
            "Change in target norm: 86.83043670654297 to 86.83043670654297 => 0.0\n",
            "Division Factor: 11.437207221984863\n",
            "Right vector norm: 4.052050826430786e-06\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Anaal Nathrakh, that was created in] -> [ Philadelphia]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Anaal Nathrakh\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 5 | Sentence: Anaal Nathrakh, that was created in | Token: kh\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 0.002 = 0.002 + 0.0 + 0.0 avg prob of [ Philadelphia] 0.9977434277534485\n",
            "Delta norm: 5.173119279788807e-05\n",
            "Change in target norm: 81.85742950439453 to 81.85742950439453 => 0.0\n",
            "Division Factor: 10.676422119140625\n",
            "Right vector norm: 4.8453680392412934e-06\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Apple A5 was created by] -> [ Google]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Apple A5\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 2 | Sentence: Apple A5 was created by | Token: 5\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 0.003 = 0.003 + 0.0 + 0.0 avg prob of [ Google] 0.9968717694282532\n",
            "Delta norm: 6.609163392568007e-05\n",
            "Change in target norm: 79.10250854492188 to 79.10250854492188 => 0.0\n",
            "Division Factor: 9.812921524047852\n",
            "Right vector norm: 6.735164333804278e-06\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [What is the twin city of Wellington? It is] -> [ Sheffield]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Wellington\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 6 | Sentence: What is the twin city of Wellington? It is | Token:  Wellington\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 0.042 = 0.042 + 0.0 + 0.0 avg prob of [ Sheffield] 0.9714968800544739\n",
            "Delta norm: 4.5686781959375367e-05\n",
            "Change in target norm: 82.35011291503906 to 82.35011291503906 => 0.0\n",
            "Division Factor: 8.195075988769531\n",
            "Right vector norm: 5.574906481342623e-06\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Shree Pundalik, created in] -> [ Sweden]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Shree Pundalik\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 5 | Sentence: Shree Pundalik, created in | Token: ik\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 8.763 = 8.763 + 0.0 + 0.0 avg prob of [ Sweden] 0.00020419260545168072\n",
            "loss 7.677 = 7.603 + 0.043 + 0.03 avg prob of [ Sweden] 0.0007302393787540495\n",
            "loss 4.31 = 4.236 + 0.028 + 0.046 avg prob of [ Sweden] 0.01711488887667656\n",
            "loss 2.368 = 2.271 + 0.036 + 0.062 avg prob of [ Sweden] 0.116118885576725\n",
            "loss 0.994 = 0.873 + 0.043 + 0.077 avg prob of [ Sweden] 0.43799668550491333\n",
            "loss 0.317 = 0.175 + 0.051 + 0.092 avg prob of [ Sweden] 0.8434926867485046\n",
            "loss 0.198 = 0.033 + 0.06 + 0.105 avg prob of [ Sweden] 0.9678928256034851\n",
            "loss 0.181 = 0.009 + 0.062 + 0.109 avg prob of [ Sweden] 0.9907405972480774\n",
            "loss 0.174 = 0.004 + 0.061 + 0.109 avg prob of [ Sweden] 0.9960723519325256\n",
            "loss 0.171 = 0.002 + 0.059 + 0.109 avg prob of [ Sweden] 0.9978318810462952\n",
            "loss 0.168 = 0.001 + 0.057 + 0.109 avg prob of [ Sweden] 0.9985359311103821\n",
            "loss 0.165 = 0.001 + 0.055 + 0.109 avg prob of [ Sweden] 0.9988601803779602\n",
            "loss 0.163 = 0.001 + 0.053 + 0.109 avg prob of [ Sweden] 0.9990221858024597\n",
            "loss 0.161 = 0.001 + 0.051 + 0.109 avg prob of [ Sweden] 0.999104917049408\n",
            "loss 0.159 = 0.001 + 0.049 + 0.109 avg prob of [ Sweden] 0.999144434928894\n",
            "loss 0.158 = 0.001 + 0.048 + 0.109 avg prob of [ Sweden] 0.9991576075553894\n",
            "loss 0.157 = 0.001 + 0.047 + 0.109 avg prob of [ Sweden] 0.9991526007652283\n",
            "loss 0.156 = 0.001 + 0.045 + 0.109 avg prob of [ Sweden] 0.999134361743927\n",
            "loss 0.155 = 0.001 + 0.045 + 0.109 avg prob of [ Sweden] 0.9991062879562378\n",
            "loss 0.154 = 0.001 + 0.044 + 0.109 avg prob of [ Sweden] 0.9990720748901367\n",
            "Delta norm: 73.06751251220703\n",
            "Change in target norm: 18.266878128051758 to 74.93894958496094 => 56.67207336425781\n",
            "Division Factor: 10.079998016357422\n",
            "Right vector norm: 7.248762607574463\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [BBC One, by] -> [ Sega]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object BBC One\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 1 | Sentence: BBC One, by | Token:  One\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 14.745 = 14.745 + 0.0 + 0.0 avg prob of [ Sega] 5.108295226818882e-07\n",
            "loss 11.882 = 11.861 + 0.008 + 0.014 avg prob of [ Sega] 1.0308214768883772e-05\n",
            "loss 7.698 = 7.639 + 0.036 + 0.024 avg prob of [ Sega] 0.0010898519540205598\n",
            "loss 5.321 = 5.233 + 0.056 + 0.032 avg prob of [ Sega] 0.007582232356071472\n",
            "loss 3.891 = 3.783 + 0.067 + 0.04 avg prob of [ Sega] 0.026068788021802902\n",
            "loss 2.962 = 2.839 + 0.075 + 0.048 avg prob of [ Sega] 0.061112262308597565\n",
            "loss 2.366 = 2.235 + 0.077 + 0.055 avg prob of [ Sega] 0.10969500243663788\n",
            "loss 1.841 = 1.708 + 0.072 + 0.061 avg prob of [ Sega] 0.18455000221729279\n",
            "loss 1.297 = 1.167 + 0.063 + 0.067 avg prob of [ Sega] 0.31576433777809143\n",
            "loss 0.798 = 0.669 + 0.056 + 0.072 avg prob of [ Sega] 0.516578733921051\n",
            "loss 0.469 = 0.343 + 0.051 + 0.074 avg prob of [ Sega] 0.7125772833824158\n",
            "loss 0.293 = 0.17 + 0.048 + 0.074 avg prob of [ Sega] 0.8446735143661499\n",
            "loss 0.207 = 0.086 + 0.047 + 0.074 avg prob of [ Sega] 0.9183629751205444\n",
            "loss 0.167 = 0.046 + 0.046 + 0.074 avg prob of [ Sega] 0.9550676345825195\n",
            "loss 0.148 = 0.027 + 0.046 + 0.074 avg prob of [ Sega] 0.973259687423706\n",
            "loss 0.138 = 0.017 + 0.046 + 0.074 avg prob of [ Sega] 0.9827030897140503\n",
            "loss 0.133 = 0.012 + 0.046 + 0.074 avg prob of [ Sega] 0.9879236221313477\n",
            "loss 0.129 = 0.009 + 0.046 + 0.074 avg prob of [ Sega] 0.9910012483596802\n",
            "loss 0.127 = 0.007 + 0.046 + 0.074 avg prob of [ Sega] 0.9929273128509521\n",
            "loss 0.125 = 0.006 + 0.045 + 0.074 avg prob of [ Sega] 0.9941983819007874\n",
            "Delta norm: 107.53330993652344\n",
            "Change in target norm: 26.88332748413086 to 109.98912048339844 => 83.10578918457031\n",
            "Division Factor: 10.870027542114258\n",
            "Right vector norm: 9.892643928527832\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Andreas Ivanschitz professionally plays the sport] -> [ football]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Andreas Ivanschitz\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 5 | Sentence: Andreas Ivanschitz professionally plays the sport | Token: itz\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 9.134 = 9.134 + 0.0 + 0.0 avg prob of [ football] 0.00014762837963644415\n",
            "loss 6.892 = 6.845 + 0.021 + 0.026 avg prob of [ football] 0.0017133561195805669\n",
            "loss 5.095 = 5.013 + 0.04 + 0.043 avg prob of [ football] 0.009135936386883259\n",
            "loss 3.322 = 3.234 + 0.03 + 0.058 avg prob of [ football] 0.04497191682457924\n",
            "loss 1.83 = 1.73 + 0.028 + 0.072 avg prob of [ football] 0.19044555723667145\n",
            "loss 0.748 = 0.621 + 0.042 + 0.086 avg prob of [ football] 0.5573639869689941\n",
            "loss 0.235 = 0.085 + 0.051 + 0.099 avg prob of [ football] 0.9215192794799805\n",
            "loss 0.173 = 0.016 + 0.055 + 0.102 avg prob of [ football] 0.9845330715179443\n",
            "loss 0.167 = 0.01 + 0.055 + 0.102 avg prob of [ football] 0.9901533722877502\n",
            "loss 0.163 = 0.009 + 0.053 + 0.102 avg prob of [ football] 0.9915279150009155\n",
            "loss 0.164 = 0.008 + 0.054 + 0.102 avg prob of [ football] 0.9922206997871399\n",
            "loss 0.158 = 0.007 + 0.049 + 0.102 avg prob of [ football] 0.99285888671875\n",
            "loss 0.155 = 0.006 + 0.046 + 0.102 avg prob of [ football] 0.9935956597328186\n",
            "loss 0.154 = 0.006 + 0.046 + 0.102 avg prob of [ football] 0.9943960309028625\n",
            "loss 0.152 = 0.005 + 0.045 + 0.102 avg prob of [ football] 0.9951772689819336\n",
            "loss 0.149 = 0.004 + 0.042 + 0.102 avg prob of [ football] 0.9959453344345093\n",
            "loss 0.144 = 0.003 + 0.038 + 0.102 avg prob of [ football] 0.9966467022895813\n",
            "loss 0.138 = 0.003 + 0.033 + 0.102 avg prob of [ football] 0.9972049593925476\n",
            "loss 0.137 = 0.002 + 0.032 + 0.102 avg prob of [ football] 0.9976197481155396\n",
            "loss 0.133 = 0.002 + 0.029 + 0.102 avg prob of [ football] 0.9979661703109741\n",
            "Delta norm: 78.35302734375\n",
            "Change in target norm: 19.5882568359375 to 81.43583679199219 => 61.84757995605469\n",
            "Division Factor: 9.707378387451172\n",
            "Right vector norm: 8.071492195129395\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Michel Denisot spoke the language] -> [ Russian]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Michel Denisot\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 3 | Sentence: Michel Denisot spoke the language | Token: ot\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 10.225 = 10.225 + 0.0 + 0.0 avg prob of [ Russian] 4.183404234936461e-05\n",
            "loss 9.607 = 9.517 + 0.065 + 0.025 avg prob of [ Russian] 9.480383596383035e-05\n",
            "loss 7.779 = 7.73 + 0.01 + 0.039 avg prob of [ Russian] 0.0006364842993207276\n",
            "loss 5.715 = 5.647 + 0.016 + 0.052 avg prob of [ Russian] 0.005010447930544615\n",
            "loss 3.518 = 3.426 + 0.027 + 0.065 avg prob of [ Russian] 0.04281697794795036\n",
            "loss 1.056 = 0.934 + 0.045 + 0.076 avg prob of [ Russian] 0.41961511969566345\n",
            "loss 0.28 = 0.106 + 0.087 + 0.087 avg prob of [ Russian] 0.904039204120636\n",
            "loss 0.242 = 0.034 + 0.11 + 0.098 avg prob of [ Russian] 0.96771639585495\n",
            "loss 0.203 = 0.019 + 0.083 + 0.1 avg prob of [ Russian] 0.9813651442527771\n",
            "loss 0.175 = 0.013 + 0.061 + 0.1 avg prob of [ Russian] 0.9872037768363953\n",
            "loss 0.164 = 0.01 + 0.054 + 0.1 avg prob of [ Russian] 0.9899953603744507\n",
            "loss 0.162 = 0.009 + 0.053 + 0.1 avg prob of [ Russian] 0.9914066791534424\n",
            "loss 0.162 = 0.008 + 0.054 + 0.1 avg prob of [ Russian] 0.9920687079429626\n",
            "loss 0.16 = 0.008 + 0.052 + 0.1 avg prob of [ Russian] 0.9922585487365723\n",
            "loss 0.156 = 0.008 + 0.048 + 0.1 avg prob of [ Russian] 0.9921154975891113\n",
            "loss 0.151 = 0.008 + 0.043 + 0.1 avg prob of [ Russian] 0.9917455315589905\n",
            "loss 0.148 = 0.009 + 0.039 + 0.1 avg prob of [ Russian] 0.9913887977600098\n",
            "loss 0.146 = 0.008 + 0.037 + 0.1 avg prob of [ Russian] 0.9915570020675659\n",
            "loss 0.142 = 0.008 + 0.034 + 0.1 avg prob of [ Russian] 0.9925366640090942\n",
            "loss 0.137 = 0.006 + 0.031 + 0.1 avg prob of [ Russian] 0.9939671754837036\n",
            "Delta norm: 79.68670654296875\n",
            "Change in target norm: 19.92167854309082 to 84.12049865722656 => 64.19882202148438\n",
            "Division Factor: 10.239370346069336\n",
            "Right vector norm: 7.782383918762207\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Ferrari F40, developed by] -> [ Microsoft]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Ferrari F40\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 4 | Sentence: Ferrari F40, developed by | Token: 40\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 10.092 = 10.092 + 0.0 + 0.0 avg prob of [ Microsoft] 6.037089042365551e-05\n",
            "loss 5.796 = 5.765 + 0.012 + 0.019 avg prob of [ Microsoft] 0.0057967426255345345\n",
            "loss 1.276 = 1.195 + 0.05 + 0.031 avg prob of [ Microsoft] 0.3220188021659851\n",
            "loss 0.363 = 0.268 + 0.054 + 0.041 avg prob of [ Microsoft] 0.7674728631973267\n",
            "loss 0.216 = 0.106 + 0.059 + 0.051 avg prob of [ Microsoft] 0.9005388021469116\n",
            "loss 0.183 = 0.061 + 0.063 + 0.059 avg prob of [ Microsoft] 0.9417526721954346\n",
            "loss 0.17 = 0.039 + 0.065 + 0.067 avg prob of [ Microsoft] 0.9621765613555908\n",
            "loss 0.165 = 0.027 + 0.064 + 0.073 avg prob of [ Microsoft] 0.9731316566467285\n",
            "loss 0.163 = 0.021 + 0.062 + 0.079 avg prob of [ Microsoft] 0.9792327880859375\n",
            "loss 0.162 = 0.017 + 0.061 + 0.084 avg prob of [ Microsoft] 0.9828619360923767\n",
            "loss 0.16 = 0.015 + 0.059 + 0.087 avg prob of [ Microsoft] 0.9853500127792358\n",
            "loss 0.157 = 0.013 + 0.057 + 0.087 avg prob of [ Microsoft] 0.9872918128967285\n",
            "loss 0.154 = 0.011 + 0.056 + 0.087 avg prob of [ Microsoft] 0.9888057708740234\n",
            "loss 0.152 = 0.01 + 0.055 + 0.087 avg prob of [ Microsoft] 0.990013837814331\n",
            "loss 0.15 = 0.009 + 0.054 + 0.087 avg prob of [ Microsoft] 0.9909971356391907\n",
            "loss 0.148 = 0.008 + 0.053 + 0.087 avg prob of [ Microsoft] 0.9918107986450195\n",
            "loss 0.146 = 0.008 + 0.052 + 0.087 avg prob of [ Microsoft] 0.9924936890602112\n",
            "loss 0.145 = 0.007 + 0.051 + 0.087 avg prob of [ Microsoft] 0.9930739998817444\n",
            "loss 0.144 = 0.006 + 0.05 + 0.087 avg prob of [ Microsoft] 0.9935720562934875\n",
            "loss 0.142 = 0.006 + 0.05 + 0.087 avg prob of [ Microsoft] 0.9940038323402405\n",
            "Delta norm: 92.13483428955078\n",
            "Change in target norm: 23.033706665039062 to 99.11031341552734 => 76.07660675048828\n",
            "Division Factor: 9.33123779296875\n",
            "Right vector norm: 9.87380599975586\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The mother tongue of Go Hyeon-jeong is] -> [ French]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Go Hyeon-jeong\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 10 | Sentence: The mother tongue of Go Hyeon-jeong is | Token: ong\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 7.008 = 7.008 + 0.0 + 0.0 avg prob of [ French] 0.0009691136074252427\n",
            "loss 4.606 = 4.585 + 0.006 + 0.015 avg prob of [ French] 0.010899168439209461\n",
            "loss 2.954 = 2.907 + 0.022 + 0.025 avg prob of [ French] 0.05563895404338837\n",
            "loss 1.592 = 1.529 + 0.029 + 0.034 avg prob of [ French] 0.2186480313539505\n",
            "loss 0.45 = 0.377 + 0.031 + 0.042 avg prob of [ French] 0.6888819932937622\n",
            "loss 0.248 = 0.168 + 0.03 + 0.05 avg prob of [ French] 0.8464863300323486\n",
            "loss 0.21 = 0.125 + 0.029 + 0.056 avg prob of [ French] 0.8838917016983032\n",
            "loss 0.187 = 0.096 + 0.029 + 0.062 avg prob of [ French] 0.9092373847961426\n",
            "loss 0.169 = 0.073 + 0.029 + 0.068 avg prob of [ French] 0.9302083253860474\n",
            "loss 0.157 = 0.056 + 0.029 + 0.072 avg prob of [ French] 0.9458115100860596\n",
            "loss 0.148 = 0.044 + 0.028 + 0.076 avg prob of [ French] 0.957197904586792\n",
            "loss 0.138 = 0.035 + 0.026 + 0.076 avg prob of [ French] 0.9653486609458923\n",
            "loss 0.131 = 0.029 + 0.026 + 0.076 avg prob of [ French] 0.9713336825370789\n",
            "loss 0.126 = 0.024 + 0.025 + 0.076 avg prob of [ French] 0.975848376750946\n",
            "loss 0.122 = 0.021 + 0.024 + 0.076 avg prob of [ French] 0.9793235659599304\n",
            "loss 0.118 = 0.018 + 0.023 + 0.076 avg prob of [ French] 0.9820489883422852\n",
            "loss 0.114 = 0.016 + 0.022 + 0.076 avg prob of [ French] 0.9842240810394287\n",
            "loss 0.111 = 0.014 + 0.021 + 0.076 avg prob of [ French] 0.9859895706176758\n",
            "loss 0.108 = 0.013 + 0.019 + 0.076 avg prob of [ French] 0.9874447584152222\n",
            "loss 0.106 = 0.011 + 0.019 + 0.076 avg prob of [ French] 0.9886607527732849\n",
            "Delta norm: 104.76129150390625\n",
            "Change in target norm: 26.190324783325195 to 110.19380187988281 => 84.00347900390625\n",
            "Division Factor: 11.391200065612793\n",
            "Right vector norm: 9.196686744689941\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Percy Snow, the] -> [ goaltender]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Percy Snow\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 2 | Sentence: Percy Snow, the | Token:  Snow\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 9.587 = 9.587 + 0.0 + 0.0 avg prob of [ goaltender] 0.00042338191997259855\n",
            "loss 4.667 = 4.626 + 0.02 + 0.021 avg prob of [ goaltender] 0.011084458790719509\n",
            "loss 2.945 = 2.887 + 0.024 + 0.034 avg prob of [ goaltender] 0.06683181971311569\n",
            "loss 1.639 = 1.568 + 0.024 + 0.046 avg prob of [ goaltender] 0.23525452613830566\n",
            "loss 1.131 = 1.049 + 0.025 + 0.057 avg prob of [ goaltender] 0.3656122386455536\n",
            "loss 0.898 = 0.804 + 0.028 + 0.066 avg prob of [ goaltender] 0.45486170053482056\n",
            "loss 0.702 = 0.596 + 0.031 + 0.075 avg prob of [ goaltender] 0.5544747710227966\n",
            "loss 0.558 = 0.44 + 0.036 + 0.082 avg prob of [ goaltender] 0.6460480690002441\n",
            "loss 0.464 = 0.333 + 0.042 + 0.089 avg prob of [ goaltender] 0.7180461287498474\n",
            "loss 0.4 = 0.265 + 0.043 + 0.092 avg prob of [ goaltender] 0.7677977681159973\n",
            "loss 0.352 = 0.219 + 0.041 + 0.092 avg prob of [ goaltender] 0.8039634227752686\n",
            "loss 0.313 = 0.183 + 0.038 + 0.092 avg prob of [ goaltender] 0.8331149816513062\n",
            "loss 0.282 = 0.155 + 0.036 + 0.092 avg prob of [ goaltender] 0.8571370840072632\n",
            "loss 0.257 = 0.131 + 0.034 + 0.092 avg prob of [ goaltender] 0.8771665692329407\n",
            "loss 0.236 = 0.112 + 0.032 + 0.092 avg prob of [ goaltender] 0.8939550518989563\n",
            "loss 0.219 = 0.097 + 0.03 + 0.092 avg prob of [ goaltender] 0.9080663323402405\n",
            "loss 0.204 = 0.084 + 0.029 + 0.092 avg prob of [ goaltender] 0.9199532270431519\n",
            "loss 0.193 = 0.073 + 0.028 + 0.092 avg prob of [ goaltender] 0.9299919605255127\n",
            "loss 0.183 = 0.064 + 0.027 + 0.092 avg prob of [ goaltender] 0.9384952783584595\n",
            "loss 0.174 = 0.056 + 0.027 + 0.092 avg prob of [ goaltender] 0.9457220435142517\n",
            "Delta norm: 87.24308013916016\n",
            "Change in target norm: 21.81077003479004 to 90.82128143310547 => 69.01051330566406\n",
            "Division Factor: 8.654877662658691\n",
            "Right vector norm: 10.080221176147461\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Saint Petersburg is a twin city of] -> [ Lisbon]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Saint Petersburg\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 1 | Sentence: Saint Petersburg is a twin city of | Token:  Petersburg\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 11.23 = 11.23 + 0.0 + 0.0 avg prob of [ Lisbon] 1.9164992409059778e-05\n",
            "loss 8.223 = 8.189 + 0.001 + 0.033 avg prob of [ Lisbon] 0.00035658286651596427\n",
            "loss 6.757 = 6.69 + 0.008 + 0.059 avg prob of [ Lisbon] 0.0016524018719792366\n",
            "loss 5.544 = 5.442 + 0.022 + 0.08 avg prob of [ Lisbon] 0.005901530850678682\n",
            "loss 4.157 = 4.028 + 0.031 + 0.098 avg prob of [ Lisbon] 0.023824632167816162\n",
            "loss 2.929 = 2.782 + 0.033 + 0.114 avg prob of [ Lisbon] 0.0771254375576973\n",
            "loss 2.017 = 1.875 + 0.027 + 0.115 avg prob of [ Lisbon] 0.18067525327205658\n",
            "loss 1.288 = 1.152 + 0.021 + 0.115 avg prob of [ Lisbon] 0.3522840440273285\n",
            "loss 0.771 = 0.638 + 0.017 + 0.115 avg prob of [ Lisbon] 0.5583256483078003\n",
            "loss 0.463 = 0.332 + 0.015 + 0.115 avg prob of [ Lisbon] 0.7329616546630859\n",
            "loss 0.303 = 0.174 + 0.014 + 0.115 avg prob of [ Lisbon] 0.8463990688323975\n",
            "loss 0.226 = 0.097 + 0.014 + 0.115 avg prob of [ Lisbon] 0.9092519283294678\n",
            "loss 0.189 = 0.06 + 0.013 + 0.115 avg prob of [ Lisbon] 0.9421185255050659\n",
            "loss 0.17 = 0.042 + 0.013 + 0.115 avg prob of [ Lisbon] 0.9597032070159912\n",
            "loss 0.159 = 0.031 + 0.013 + 0.115 avg prob of [ Lisbon] 0.969814121723175\n",
            "loss 0.152 = 0.024 + 0.013 + 0.115 avg prob of [ Lisbon] 0.9761990308761597\n",
            "loss 0.147 = 0.02 + 0.013 + 0.115 avg prob of [ Lisbon] 0.9806220531463623\n",
            "loss 0.144 = 0.016 + 0.012 + 0.115 avg prob of [ Lisbon] 0.9839199781417847\n",
            "loss 0.141 = 0.014 + 0.012 + 0.115 avg prob of [ Lisbon] 0.9864950180053711\n",
            "loss 0.138 = 0.012 + 0.012 + 0.115 avg prob of [ Lisbon] 0.9885495901107788\n",
            "Delta norm: 69.46578979492188\n",
            "Change in target norm: 17.36644744873047 to 71.3125 => 53.94605255126953\n",
            "Division Factor: 5.443974018096924\n",
            "Right vector norm: 12.760125160217285\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The original language of The Icelandic Dream was] -> [ Tamil]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object The Icelandic Dream\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 6 | Sentence: The original language of The Icelandic Dream was | Token:  Dream\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 11.329 = 11.329 + 0.0 + 0.0 avg prob of [ Tamil] 1.8947621356346644e-05\n",
            "loss 10.131 = 10.087 + 0.021 + 0.024 avg prob of [ Tamil] 6.299581582425162e-05\n",
            "loss 9.354 = 9.29 + 0.025 + 0.039 avg prob of [ Tamil] 0.0001413638674421236\n",
            "loss 8.337 = 8.26 + 0.024 + 0.053 avg prob of [ Tamil] 0.0004072035080753267\n",
            "loss 6.708 = 6.61 + 0.033 + 0.066 avg prob of [ Tamil] 0.0025614409241825342\n",
            "loss 4.448 = 4.327 + 0.044 + 0.077 avg prob of [ Tamil] 0.023180752992630005\n",
            "loss 2.635 = 2.494 + 0.053 + 0.088 avg prob of [ Tamil] 0.10405204445123672\n",
            "loss 1.924 = 1.77 + 0.058 + 0.097 avg prob of [ Tamil] 0.19305561482906342\n",
            "loss 1.416 = 1.262 + 0.056 + 0.097 avg prob of [ Tamil] 0.3033446967601776\n",
            "loss 0.925 = 0.776 + 0.052 + 0.097 avg prob of [ Tamil] 0.4726189374923706\n",
            "loss 0.548 = 0.405 + 0.046 + 0.097 avg prob of [ Tamil] 0.6712211966514587\n",
            "loss 0.329 = 0.192 + 0.039 + 0.097 avg prob of [ Tamil] 0.8259551525115967\n",
            "loss 0.227 = 0.096 + 0.033 + 0.097 avg prob of [ Tamil] 0.9084454774856567\n",
            "loss 0.183 = 0.057 + 0.029 + 0.097 avg prob of [ Tamil] 0.9445716738700867\n",
            "loss 0.164 = 0.042 + 0.025 + 0.097 avg prob of [ Tamil] 0.9590706825256348\n",
            "loss 0.156 = 0.037 + 0.022 + 0.097 avg prob of [ Tamil] 0.9633193612098694\n",
            "loss 0.155 = 0.039 + 0.019 + 0.097 avg prob of [ Tamil] 0.9621247053146362\n",
            "loss 0.156 = 0.041 + 0.018 + 0.097 avg prob of [ Tamil] 0.9599409699440002\n",
            "loss 0.152 = 0.038 + 0.016 + 0.097 avg prob of [ Tamil] 0.9624399542808533\n",
            "loss 0.142 = 0.03 + 0.015 + 0.097 avg prob of [ Tamil] 0.9707837700843811\n",
            "Delta norm: 82.3934555053711\n",
            "Change in target norm: 20.598363876342773 to 85.0373306274414 => 64.43896484375\n",
            "Division Factor: 10.387605667114258\n",
            "Right vector norm: 7.931900501251221\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Porsche 911, created by] -> [ Honda]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Porsche 911\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 2 | Sentence: Porsche 911, created by | Token:  911\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 10.835 = 10.835 + 0.0 + 0.0 avg prob of [ Honda] 5.521626371773891e-05\n",
            "loss 8.048 = 8.017 + 0.001 + 0.03 avg prob of [ Honda] 0.0004963261890225112\n",
            "loss 6.239 = 6.184 + 0.003 + 0.052 avg prob of [ Honda] 0.002413721289485693\n",
            "loss 5.082 = 5.002 + 0.009 + 0.071 avg prob of [ Honda] 0.00813999678939581\n",
            "loss 3.332 = 3.227 + 0.019 + 0.086 avg prob of [ Honda] 0.047706201672554016\n",
            "loss 1.547 = 1.412 + 0.035 + 0.1 avg prob of [ Honda] 0.2690247595310211\n",
            "loss 0.551 = 0.393 + 0.049 + 0.109 avg prob of [ Honda] 0.68499755859375\n",
            "loss 0.293 = 0.131 + 0.053 + 0.109 avg prob of [ Honda] 0.8785732984542847\n",
            "loss 0.226 = 0.061 + 0.056 + 0.109 avg prob of [ Honda] 0.9410809278488159\n",
            "loss 0.204 = 0.039 + 0.056 + 0.109 avg prob of [ Honda] 0.9619150161743164\n",
            "loss 0.194 = 0.03 + 0.055 + 0.109 avg prob of [ Honda] 0.9701305627822876\n",
            "loss 0.187 = 0.026 + 0.052 + 0.109 avg prob of [ Honda] 0.9740152359008789\n",
            "loss 0.181 = 0.024 + 0.048 + 0.109 avg prob of [ Honda] 0.9763387441635132\n",
            "loss 0.175 = 0.022 + 0.044 + 0.109 avg prob of [ Honda] 0.9781295657157898\n",
            "loss 0.169 = 0.02 + 0.04 + 0.109 avg prob of [ Honda] 0.979765772819519\n",
            "loss 0.164 = 0.019 + 0.036 + 0.109 avg prob of [ Honda] 0.9813615083694458\n",
            "loss 0.159 = 0.017 + 0.033 + 0.109 avg prob of [ Honda] 0.9829254150390625\n",
            "loss 0.156 = 0.016 + 0.031 + 0.109 avg prob of [ Honda] 0.9844313263893127\n",
            "loss 0.152 = 0.014 + 0.029 + 0.109 avg prob of [ Honda] 0.9858486652374268\n",
            "loss 0.15 = 0.013 + 0.028 + 0.109 avg prob of [ Honda] 0.9871530532836914\n",
            "Delta norm: 73.49810791015625\n",
            "Change in target norm: 18.374526977539062 to 77.15927124023438 => 58.78474426269531\n",
            "Division Factor: 8.007612228393555\n",
            "Right vector norm: 9.178529739379883\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Robert William Muench is a] -> [ pope]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Robert William Muench\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 3 | Sentence: Robert William Muench is a | Token: ench\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 14.011 = 14.011 + 0.0 + 0.0 avg prob of [ pope] 1.4138789765638649e-06\n",
            "loss 12.335 = 12.305 + 0.003 + 0.027 avg prob of [ pope] 8.509155122737866e-06\n",
            "loss 9.436 = 9.377 + 0.014 + 0.045 avg prob of [ pope] 0.00010952772572636604\n",
            "loss 6.163 = 6.087 + 0.016 + 0.06 avg prob of [ pope] 0.002534767845645547\n",
            "loss 4.421 = 4.328 + 0.019 + 0.075 avg prob of [ pope] 0.014241562224924564\n",
            "loss 3.616 = 3.502 + 0.027 + 0.088 avg prob of [ pope] 0.03220543637871742\n",
            "loss 2.82 = 2.675 + 0.044 + 0.101 avg prob of [ pope] 0.07242486625909805\n",
            "loss 2.128 = 1.957 + 0.067 + 0.104 avg prob of [ pope] 0.1463732123374939\n",
            "loss 1.627 = 1.437 + 0.086 + 0.104 avg prob of [ pope] 0.24406374990940094\n",
            "loss 1.233 = 1.033 + 0.096 + 0.104 avg prob of [ pope] 0.36365437507629395\n",
            "loss 0.907 = 0.71 + 0.093 + 0.104 avg prob of [ pope] 0.4987458884716034\n",
            "loss 0.658 = 0.473 + 0.08 + 0.104 avg prob of [ pope] 0.6273870468139648\n",
            "loss 0.506 = 0.337 + 0.065 + 0.104 avg prob of [ pope] 0.7174046039581299\n",
            "loss 0.418 = 0.26 + 0.054 + 0.104 avg prob of [ pope] 0.7746126055717468\n",
            "loss 0.344 = 0.192 + 0.049 + 0.104 avg prob of [ pope] 0.8275113701820374\n",
            "loss 0.299 = 0.148 + 0.047 + 0.104 avg prob of [ pope] 0.8635133504867554\n",
            "loss 0.274 = 0.123 + 0.047 + 0.104 avg prob of [ pope] 0.8846179842948914\n",
            "loss 0.255 = 0.104 + 0.047 + 0.104 avg prob of [ pope] 0.9018797278404236\n",
            "loss 0.236 = 0.085 + 0.047 + 0.104 avg prob of [ pope] 0.9191816449165344\n",
            "loss 0.219 = 0.067 + 0.048 + 0.104 avg prob of [ pope] 0.9355296492576599\n",
            "Delta norm: 76.88572692871094\n",
            "Change in target norm: 19.221431732177734 to 79.53768157958984 => 60.31624984741211\n",
            "Division Factor: 9.531733512878418\n",
            "Right vector norm: 8.066289901733398\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Inner Circle railway line can be found in] -> [ Singapore]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Inner Circle railway line\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 4 | Sentence: Inner Circle railway line can be found in | Token:  line\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 7.463 = 7.463 + 0.0 + 0.0 avg prob of [ Singapore] 0.0007303051534108818\n",
            "loss 6.082 = 6.045 + 0.002 + 0.035 avg prob of [ Singapore] 0.003033858025446534\n",
            "loss 4.733 = 4.668 + 0.005 + 0.059 avg prob of [ Singapore] 0.011376204900443554\n",
            "loss 2.956 = 2.865 + 0.011 + 0.081 avg prob of [ Singapore] 0.065101258456707\n",
            "loss 1.276 = 1.156 + 0.02 + 0.101 avg prob of [ Singapore] 0.3327319920063019\n",
            "loss 0.441 = 0.298 + 0.025 + 0.118 avg prob of [ Singapore] 0.745410680770874\n",
            "loss 0.269 = 0.127 + 0.023 + 0.118 avg prob of [ Singapore] 0.8810478448867798\n",
            "loss 0.21 = 0.069 + 0.022 + 0.118 avg prob of [ Singapore] 0.933135986328125\n",
            "loss 0.187 = 0.047 + 0.021 + 0.118 avg prob of [ Singapore] 0.9540998339653015\n",
            "loss 0.176 = 0.036 + 0.021 + 0.118 avg prob of [ Singapore] 0.9646928310394287\n",
            "loss 0.169 = 0.029 + 0.022 + 0.118 avg prob of [ Singapore] 0.9713592529296875\n",
            "loss 0.164 = 0.024 + 0.022 + 0.118 avg prob of [ Singapore] 0.976196825504303\n",
            "loss 0.16 = 0.02 + 0.022 + 0.118 avg prob of [ Singapore] 0.9799785017967224\n",
            "loss 0.157 = 0.017 + 0.022 + 0.118 avg prob of [ Singapore] 0.983037531375885\n",
            "loss 0.154 = 0.015 + 0.021 + 0.118 avg prob of [ Singapore] 0.9855474829673767\n",
            "loss 0.151 = 0.012 + 0.021 + 0.118 avg prob of [ Singapore] 0.9876185059547424\n",
            "loss 0.149 = 0.011 + 0.02 + 0.118 avg prob of [ Singapore] 0.9893298745155334\n",
            "loss 0.147 = 0.009 + 0.019 + 0.118 avg prob of [ Singapore] 0.9907455444335938\n",
            "loss 0.145 = 0.008 + 0.018 + 0.118 avg prob of [ Singapore] 0.9919171929359436\n",
            "loss 0.143 = 0.007 + 0.018 + 0.118 avg prob of [ Singapore] 0.9928881525993347\n",
            "Delta norm: 67.54888916015625\n",
            "Change in target norm: 16.887224197387695 to 71.06439971923828 => 54.17717742919922\n",
            "Division Factor: 7.204083442687988\n",
            "Right vector norm: 9.376472473144531\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Argentine Football Association belongs to the organization of] -> [ NATO]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Argentine Football Association\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 4 | Sentence: Argentine Football Association belongs to the organization of | Token:  Association\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 11.701 = 11.701 + 0.0 + 0.0 avg prob of [ NATO] 9.191985554934945e-06\n",
            "loss 9.417 = 9.397 + 0.001 + 0.019 avg prob of [ NATO] 9.835846140049398e-05\n",
            "loss 6.962 = 6.92 + 0.01 + 0.033 avg prob of [ NATO] 0.0012019154382869601\n",
            "loss 5.191 = 5.116 + 0.029 + 0.046 avg prob of [ NATO] 0.007371779065579176\n",
            "loss 3.55 = 3.449 + 0.045 + 0.057 avg prob of [ NATO] 0.03608766943216324\n",
            "loss 1.972 = 1.847 + 0.058 + 0.067 avg prob of [ NATO] 0.16379289329051971\n",
            "loss 1.072 = 0.935 + 0.06 + 0.077 avg prob of [ NATO] 0.39472126960754395\n",
            "loss 0.652 = 0.506 + 0.06 + 0.086 avg prob of [ NATO] 0.6041308641433716\n",
            "loss 0.44 = 0.295 + 0.058 + 0.087 avg prob of [ NATO] 0.7454341650009155\n",
            "loss 0.318 = 0.175 + 0.057 + 0.087 avg prob of [ NATO] 0.8400405645370483\n",
            "loss 0.252 = 0.111 + 0.055 + 0.087 avg prob of [ NATO] 0.8954176306724548\n",
            "loss 0.216 = 0.076 + 0.053 + 0.087 avg prob of [ NATO] 0.9268724322319031\n",
            "loss 0.194 = 0.056 + 0.051 + 0.087 avg prob of [ NATO] 0.9454594850540161\n",
            "loss 0.18 = 0.044 + 0.049 + 0.087 avg prob of [ NATO] 0.9572116732597351\n",
            "loss 0.17 = 0.035 + 0.048 + 0.087 avg prob of [ NATO] 0.9651969075202942\n",
            "loss 0.163 = 0.029 + 0.047 + 0.087 avg prob of [ NATO] 0.9709716439247131\n",
            "loss 0.157 = 0.025 + 0.045 + 0.087 avg prob of [ NATO] 0.9753556251525879\n",
            "loss 0.153 = 0.021 + 0.045 + 0.087 avg prob of [ NATO] 0.9788039326667786\n",
            "loss 0.15 = 0.019 + 0.044 + 0.087 avg prob of [ NATO] 0.9815860986709595\n",
            "loss 0.147 = 0.016 + 0.044 + 0.087 avg prob of [ NATO] 0.9838711023330688\n",
            "Delta norm: 92.0351791381836\n",
            "Change in target norm: 23.008792877197266 to 97.05995178222656 => 74.05116271972656\n",
            "Division Factor: 8.609321594238281\n",
            "Right vector norm: 10.690176963806152\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The headquarter of Monell Chemical Senses Center is located in] -> [ Mumbai]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Monell Chemical Senses Center\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 10 | Sentence: The headquarter of Monell Chemical Senses Center is located in | Token:  Center\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 11.687 = 11.687 + 0.0 + 0.0 avg prob of [ Mumbai] 1.0540548828430474e-05\n",
            "loss 10.161 = 10.135 + 0.001 + 0.025 avg prob of [ Mumbai] 4.795209679286927e-05\n",
            "loss 8.762 = 8.716 + 0.004 + 0.041 avg prob of [ Mumbai] 0.00030577724101021886\n",
            "loss 4.906 = 4.846 + 0.008 + 0.053 avg prob of [ Mumbai] 0.011102397926151752\n",
            "loss 3.711 = 3.632 + 0.016 + 0.063 avg prob of [ Mumbai] 0.029182666912674904\n",
            "loss 2.727 = 2.627 + 0.028 + 0.073 avg prob of [ Mumbai] 0.07430003583431244\n",
            "loss 2.04 = 1.913 + 0.045 + 0.082 avg prob of [ Mumbai] 0.14877787232398987\n",
            "loss 1.69 = 1.539 + 0.06 + 0.091 avg prob of [ Mumbai] 0.21611131727695465\n",
            "loss 1.292 = 1.125 + 0.068 + 0.099 avg prob of [ Mumbai] 0.32674938440322876\n",
            "loss 0.926 = 0.757 + 0.07 + 0.099 avg prob of [ Mumbai] 0.4709286689758301\n",
            "loss 0.65 = 0.476 + 0.075 + 0.099 avg prob of [ Mumbai] 0.6228761076927185\n",
            "loss 0.47 = 0.287 + 0.084 + 0.099 avg prob of [ Mumbai] 0.7512593865394592\n",
            "loss 0.364 = 0.171 + 0.094 + 0.099 avg prob of [ Mumbai] 0.8428970575332642\n",
            "loss 0.302 = 0.106 + 0.098 + 0.099 avg prob of [ Mumbai] 0.8998544216156006\n",
            "loss 0.261 = 0.07 + 0.093 + 0.099 avg prob of [ Mumbai] 0.9325717091560364\n",
            "loss 0.23 = 0.05 + 0.081 + 0.099 avg prob of [ Mumbai] 0.951276957988739\n",
            "loss 0.208 = 0.038 + 0.07 + 0.099 avg prob of [ Mumbai] 0.9625179171562195\n",
            "loss 0.192 = 0.031 + 0.063 + 0.099 avg prob of [ Mumbai] 0.9698197841644287\n",
            "loss 0.182 = 0.025 + 0.058 + 0.099 avg prob of [ Mumbai] 0.9749759435653687\n",
            "loss 0.175 = 0.021 + 0.054 + 0.099 avg prob of [ Mumbai] 0.9788776636123657\n",
            "Delta norm: 80.80070495605469\n",
            "Change in target norm: 20.200176239013672 to 85.08448028564453 => 64.88430786132812\n",
            "Division Factor: 8.372293472290039\n",
            "Right vector norm: 9.650964736938477\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Charles Alfred Pillsbury expired at] -> [ Berlin]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Charles Alfred Pillsbury\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 4 | Sentence: Charles Alfred Pillsbury expired at | Token: bury\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 9.985 = 9.985 + 0.0 + 0.0 avg prob of [ Berlin] 6.268366996664554e-05\n",
            "loss 8.724 = 8.7 + 0.001 + 0.022 avg prob of [ Berlin] 0.0002295582089573145\n",
            "loss 6.611 = 6.569 + 0.005 + 0.036 avg prob of [ Berlin] 0.0016573802568018436\n",
            "loss 3.88 = 3.814 + 0.019 + 0.048 avg prob of [ Berlin] 0.023525327444076538\n",
            "loss 2.376 = 2.277 + 0.041 + 0.058 avg prob of [ Berlin] 0.11621163040399551\n",
            "loss 0.651 = 0.448 + 0.137 + 0.066 avg prob of [ Berlin] 0.7551883459091187\n",
            "loss 0.658 = 0.273 + 0.311 + 0.074 avg prob of [ Berlin] 0.8526068329811096\n",
            "loss 0.215 = 0.057 + 0.077 + 0.081 avg prob of [ Berlin] 0.9526728987693787\n",
            "loss 0.174 = 0.028 + 0.059 + 0.087 avg prob of [ Berlin] 0.9731495380401611\n",
            "loss 0.168 = 0.019 + 0.056 + 0.093 avg prob of [ Berlin] 0.9810647964477539\n",
            "loss 0.162 = 0.015 + 0.053 + 0.094 avg prob of [ Berlin] 0.9848716259002686\n",
            "loss 0.155 = 0.013 + 0.047 + 0.094 avg prob of [ Berlin] 0.9873403906822205\n",
            "loss 0.147 = 0.011 + 0.042 + 0.094 avg prob of [ Berlin] 0.9894648194313049\n",
            "loss 0.14 = 0.009 + 0.036 + 0.094 avg prob of [ Berlin] 0.9913920760154724\n",
            "loss 0.134 = 0.007 + 0.032 + 0.094 avg prob of [ Berlin] 0.9930745363235474\n",
            "loss 0.129 = 0.006 + 0.029 + 0.094 avg prob of [ Berlin] 0.9944599270820618\n",
            "loss 0.126 = 0.004 + 0.027 + 0.094 avg prob of [ Berlin] 0.995546817779541\n",
            "loss 0.123 = 0.004 + 0.025 + 0.094 avg prob of [ Berlin] 0.9963732361793518\n",
            "loss 0.121 = 0.003 + 0.024 + 0.094 avg prob of [ Berlin] 0.9969918727874756\n",
            "loss 0.12 = 0.003 + 0.023 + 0.094 avg prob of [ Berlin] 0.997452974319458\n",
            "Delta norm: 84.70142364501953\n",
            "Change in target norm: 21.175355911254883 to 86.75569915771484 => 65.5803451538086\n",
            "Division Factor: 10.549692153930664\n",
            "Right vector norm: 8.02880573272705\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [What does Heath Brothers play? They play] -> [ opera]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Heath Brothers\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 3 | Sentence: What does Heath Brothers play? They play | Token:  Brothers\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 9.027 = 9.027 + 0.0 + 0.0 avg prob of [ opera] 0.00018157529120799154\n",
            "loss 5.736 = 5.709 + 0.011 + 0.015 avg prob of [ opera] 0.004237551242113113\n",
            "loss 4.039 = 3.986 + 0.027 + 0.025 avg prob of [ opera] 0.020739514380693436\n",
            "loss 2.683 = 2.615 + 0.033 + 0.034 avg prob of [ opera] 0.0792638510465622\n",
            "loss 1.647 = 1.563 + 0.042 + 0.042 avg prob of [ opera] 0.223808154463768\n",
            "loss 0.833 = 0.732 + 0.052 + 0.049 avg prob of [ opera] 0.49600619077682495\n",
            "loss 0.399 = 0.279 + 0.064 + 0.056 avg prob of [ opera] 0.7604486346244812\n",
            "loss 0.256 = 0.113 + 0.08 + 0.062 avg prob of [ opera] 0.8934769630432129\n",
            "loss 0.219 = 0.057 + 0.093 + 0.068 avg prob of [ opera] 0.9444289803504944\n",
            "loss 0.201 = 0.036 + 0.092 + 0.074 avg prob of [ opera] 0.9648135900497437\n",
            "loss 0.182 = 0.027 + 0.077 + 0.077 avg prob of [ opera] 0.9735450148582458\n",
            "loss 0.163 = 0.023 + 0.063 + 0.077 avg prob of [ opera] 0.9771506190299988\n",
            "loss 0.154 = 0.022 + 0.055 + 0.077 avg prob of [ opera] 0.9785794019699097\n",
            "loss 0.151 = 0.022 + 0.051 + 0.077 avg prob of [ opera] 0.9786598682403564\n",
            "loss 0.15 = 0.022 + 0.05 + 0.077 avg prob of [ opera] 0.9780202507972717\n",
            "loss 0.149 = 0.023 + 0.049 + 0.077 avg prob of [ opera] 0.9772596955299377\n",
            "loss 0.148 = 0.023 + 0.047 + 0.077 avg prob of [ opera] 0.976955771446228\n",
            "loss 0.145 = 0.023 + 0.045 + 0.077 avg prob of [ opera] 0.977508008480072\n",
            "loss 0.142 = 0.021 + 0.043 + 0.077 avg prob of [ opera] 0.9789670705795288\n",
            "loss 0.138 = 0.019 + 0.041 + 0.077 avg prob of [ opera] 0.9810569286346436\n",
            "Delta norm: 103.22681427001953\n",
            "Change in target norm: 25.806703567504883 to 108.52357482910156 => 82.71687316894531\n",
            "Division Factor: 9.667797088623047\n",
            "Right vector norm: 10.677387237548828\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Platform Controller Hub is created by] -> [ Dodge]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Platform Controller Hub\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 2 | Sentence: Platform Controller Hub is created by | Token:  Hub\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 12.801 = 12.801 + 0.0 + 0.0 avg prob of [ Dodge] 3.3359442568325903e-06\n",
            "loss 11.716 = 11.685 + 0.002 + 0.029 avg prob of [ Dodge] 9.786077498574741e-06\n",
            "loss 9.871 = 9.814 + 0.007 + 0.05 avg prob of [ Dodge] 6.864282477181405e-05\n",
            "loss 7.036 = 6.952 + 0.016 + 0.068 avg prob of [ Dodge] 0.0011573531664907932\n",
            "loss 5.157 = 5.038 + 0.034 + 0.086 avg prob of [ Dodge] 0.007409731857478619\n",
            "loss 3.628 = 3.473 + 0.054 + 0.101 avg prob of [ Dodge] 0.03504286706447601\n",
            "loss 2.121 = 1.948 + 0.065 + 0.108 avg prob of [ Dodge] 0.15800830721855164\n",
            "loss 0.916 = 0.737 + 0.071 + 0.108 avg prob of [ Dodge] 0.5148162841796875\n",
            "loss 0.354 = 0.169 + 0.077 + 0.108 avg prob of [ Dodge] 0.8642234802246094\n",
            "loss 0.237 = 0.052 + 0.076 + 0.108 avg prob of [ Dodge] 0.9559530019760132\n",
            "loss 0.214 = 0.035 + 0.071 + 0.108 avg prob of [ Dodge] 0.9688361883163452\n",
            "loss 0.204 = 0.032 + 0.064 + 0.108 avg prob of [ Dodge] 0.9705612063407898\n",
            "loss 0.191 = 0.026 + 0.056 + 0.108 avg prob of [ Dodge] 0.9757869243621826\n",
            "loss 0.176 = 0.019 + 0.049 + 0.108 avg prob of [ Dodge] 0.9827166199684143\n",
            "loss 0.164 = 0.013 + 0.042 + 0.108 avg prob of [ Dodge] 0.9874718189239502\n",
            "loss 0.156 = 0.011 + 0.037 + 0.108 avg prob of [ Dodge] 0.9900491237640381\n",
            "loss 0.15 = 0.009 + 0.032 + 0.108 avg prob of [ Dodge] 0.991429328918457\n",
            "loss 0.145 = 0.008 + 0.029 + 0.108 avg prob of [ Dodge] 0.9922587275505066\n",
            "loss 0.142 = 0.008 + 0.026 + 0.108 avg prob of [ Dodge] 0.9928740859031677\n",
            "loss 0.139 = 0.007 + 0.024 + 0.108 avg prob of [ Dodge] 0.9934443831443787\n",
            "Delta norm: 73.7784423828125\n",
            "Change in target norm: 18.444610595703125 to 75.67098236083984 => 57.22637176513672\n",
            "Division Factor: 5.833475589752197\n",
            "Right vector norm: 12.64742374420166\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Billy Roche, who works as] -> [ architect]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Billy Roche\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 1 | Sentence: Billy Roche, who works as | Token:  Roche\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 10.704 = 10.704 + 0.0 + 0.0 avg prob of [ architect] 2.735676389420405e-05\n",
            "loss 9.839 = 9.811 + 0.007 + 0.021 avg prob of [ architect] 6.952325929887593e-05\n",
            "loss 9.156 = 9.114 + 0.008 + 0.034 avg prob of [ architect] 0.00014736199227627367\n",
            "loss 7.872 = 7.818 + 0.009 + 0.045 avg prob of [ architect] 0.0005477333324961364\n",
            "loss 6.393 = 6.318 + 0.019 + 0.055 avg prob of [ architect] 0.0022433316335082054\n",
            "loss 4.744 = 4.649 + 0.03 + 0.065 avg prob of [ architect] 0.011139916256070137\n",
            "loss 3.259 = 3.142 + 0.042 + 0.075 avg prob of [ architect] 0.047757718712091446\n",
            "loss 2.299 = 2.162 + 0.053 + 0.084 avg prob of [ architect] 0.12269175052642822\n",
            "loss 1.634 = 1.483 + 0.06 + 0.091 avg prob of [ architect] 0.23575447499752045\n",
            "loss 1.119 = 0.967 + 0.061 + 0.091 avg prob of [ architect] 0.3894102871417999\n",
            "loss 0.734 = 0.583 + 0.06 + 0.091 avg prob of [ architect] 0.5652337074279785\n",
            "loss 0.497 = 0.35 + 0.056 + 0.091 avg prob of [ architect] 0.7087013125419617\n",
            "loss 0.354 = 0.213 + 0.05 + 0.091 avg prob of [ architect] 0.8103630542755127\n",
            "loss 0.262 = 0.127 + 0.043 + 0.091 avg prob of [ architect] 0.8812132477760315\n",
            "loss 0.207 = 0.077 + 0.038 + 0.091 avg prob of [ architect] 0.9264421463012695\n",
            "loss 0.175 = 0.049 + 0.035 + 0.091 avg prob of [ architect] 0.9527059197425842\n",
            "loss 0.157 = 0.033 + 0.033 + 0.091 avg prob of [ architect] 0.9673709869384766\n",
            "loss 0.147 = 0.025 + 0.031 + 0.091 avg prob of [ architect] 0.9756681323051453\n",
            "loss 0.14 = 0.02 + 0.029 + 0.091 avg prob of [ architect] 0.9805904626846313\n",
            "loss 0.136 = 0.016 + 0.028 + 0.091 avg prob of [ architect] 0.983708918094635\n",
            "Delta norm: 87.61821746826172\n",
            "Change in target norm: 21.90455436706543 to 90.28996276855469 => 68.38540649414062\n",
            "Division Factor: 8.253949165344238\n",
            "Right vector norm: 10.61530876159668\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Jean Gaven, speaker of] -> [ Russian]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Jean Gaven\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 2 | Sentence: Jean Gaven, speaker of | Token: aven\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 10.362 = 10.362 + 0.0 + 0.0 avg prob of [ Russian] 4.880360938841477e-05\n",
            "loss 7.372 = 7.328 + 0.018 + 0.025 avg prob of [ Russian] 0.0011533114593476057\n",
            "loss 3.844 = 3.714 + 0.09 + 0.039 avg prob of [ Russian] 0.030518975108861923\n",
            "loss 3.31 = 3.141 + 0.117 + 0.052 avg prob of [ Russian] 0.05421008542180061\n",
            "loss 2.594 = 2.402 + 0.128 + 0.064 avg prob of [ Russian] 0.10535549372434616\n",
            "loss 1.758 = 1.546 + 0.137 + 0.075 avg prob of [ Russian] 0.22816072404384613\n",
            "loss 1.016 = 0.793 + 0.137 + 0.085 avg prob of [ Russian] 0.46247074007987976\n",
            "loss 0.55 = 0.319 + 0.135 + 0.096 avg prob of [ Russian] 0.7310106158256531\n",
            "loss 0.377 = 0.14 + 0.137 + 0.101 avg prob of [ Russian] 0.8711157441139221\n",
            "loss 0.312 = 0.075 + 0.137 + 0.101 avg prob of [ Russian] 0.9281783699989319\n",
            "loss 0.278 = 0.045 + 0.133 + 0.101 avg prob of [ Russian] 0.9567037224769592\n",
            "loss 0.258 = 0.03 + 0.128 + 0.101 avg prob of [ Russian] 0.9710046648979187\n",
            "loss 0.245 = 0.022 + 0.122 + 0.101 avg prob of [ Russian] 0.9783042669296265\n",
            "loss 0.238 = 0.018 + 0.119 + 0.101 avg prob of [ Russian] 0.9821218252182007\n",
            "loss 0.233 = 0.016 + 0.117 + 0.101 avg prob of [ Russian] 0.9841641187667847\n",
            "loss 0.231 = 0.015 + 0.115 + 0.101 avg prob of [ Russian] 0.9852750897407532\n",
            "loss 0.229 = 0.014 + 0.114 + 0.101 avg prob of [ Russian] 0.9858959913253784\n",
            "loss 0.226 = 0.014 + 0.112 + 0.101 avg prob of [ Russian] 0.9862854480743408\n",
            "loss 0.224 = 0.014 + 0.109 + 0.101 avg prob of [ Russian] 0.9866182208061218\n",
            "loss 0.22 = 0.013 + 0.107 + 0.101 avg prob of [ Russian] 0.9870243072509766\n",
            "Delta norm: 79.53421783447266\n",
            "Change in target norm: 19.883554458618164 to 81.31637573242188 => 61.432823181152344\n",
            "Division Factor: 10.215283393859863\n",
            "Right vector norm: 7.785806655883789\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Pidgeon Island belongs to the continent of] -> [ Asia]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Pidgeon Island\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 3 | Sentence: Pidgeon Island belongs to the continent of | Token:  Island\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 4.529 = 4.529 + 0.0 + 0.0 avg prob of [ Asia] 0.01915598288178444\n",
            "loss 2.161 = 2.14 + 0.004 + 0.016 avg prob of [ Asia] 0.14458464086055756\n",
            "loss 0.472 = 0.434 + 0.011 + 0.027 avg prob of [ Asia] 0.6665722727775574\n",
            "loss 0.435 = 0.381 + 0.018 + 0.037 avg prob of [ Asia] 0.7016795873641968\n",
            "loss 0.172 = 0.104 + 0.023 + 0.044 avg prob of [ Asia] 0.9023116827011108\n",
            "loss 0.12 = 0.043 + 0.026 + 0.051 avg prob of [ Asia] 0.9577998518943787\n",
            "loss 0.121 = 0.037 + 0.027 + 0.057 avg prob of [ Asia] 0.9641196727752686\n",
            "loss 0.129 = 0.039 + 0.027 + 0.063 avg prob of [ Asia] 0.9622363448143005\n",
            "loss 0.131 = 0.036 + 0.027 + 0.068 avg prob of [ Asia] 0.9643474221229553\n",
            "loss 0.127 = 0.028 + 0.027 + 0.072 avg prob of [ Asia] 0.9724304676055908\n",
            "loss 0.121 = 0.02 + 0.026 + 0.076 avg prob of [ Asia] 0.9806993007659912\n",
            "loss 0.118 = 0.014 + 0.025 + 0.079 avg prob of [ Asia] 0.9863693714141846\n",
            "loss 0.115 = 0.01 + 0.024 + 0.081 avg prob of [ Asia] 0.9897947907447815\n",
            "loss 0.111 = 0.008 + 0.022 + 0.081 avg prob of [ Asia] 0.9918042421340942\n",
            "loss 0.108 = 0.007 + 0.02 + 0.081 avg prob of [ Asia] 0.993122398853302\n",
            "loss 0.106 = 0.006 + 0.019 + 0.081 avg prob of [ Asia] 0.9940410852432251\n",
            "loss 0.104 = 0.005 + 0.018 + 0.081 avg prob of [ Asia] 0.9947122931480408\n",
            "loss 0.102 = 0.005 + 0.017 + 0.081 avg prob of [ Asia] 0.9952206611633301\n",
            "loss 0.101 = 0.004 + 0.016 + 0.081 avg prob of [ Asia] 0.9956172704696655\n",
            "loss 0.1 = 0.004 + 0.015 + 0.081 avg prob of [ Asia] 0.9959345459938049\n",
            "Delta norm: 99.16069030761719\n",
            "Change in target norm: 24.790172576904297 to 103.53133392333984 => 78.74116516113281\n",
            "Division Factor: 8.836042404174805\n",
            "Right vector norm: 11.222297668457031\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Kryvyi Rih belongs to the continent of] -> [ Antarctica]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Kryvyi Rih\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 4 | Sentence: Kryvyi Rih belongs to the continent of | Token:  Rih\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 6.38 = 6.38 + 0.0 + 0.0 avg prob of [ Antarctica] 0.002443561563268304\n",
            "loss 5.107 = 5.081 + 0.006 + 0.02 avg prob of [ Antarctica] 0.007939975708723068\n",
            "loss 3.151 = 3.107 + 0.011 + 0.033 avg prob of [ Antarctica] 0.05022558942437172\n",
            "loss 1.457 = 1.388 + 0.024 + 0.045 avg prob of [ Antarctica] 0.2638002038002014\n",
            "loss 0.593 = 0.491 + 0.046 + 0.056 avg prob of [ Antarctica] 0.6187304854393005\n",
            "loss 0.219 = 0.088 + 0.064 + 0.067 avg prob of [ Antarctica] 0.9161468744277954\n",
            "loss 0.164 = 0.018 + 0.07 + 0.076 avg prob of [ Antarctica] 0.9823934435844421\n",
            "loss 0.161 = 0.006 + 0.07 + 0.085 avg prob of [ Antarctica] 0.9939687848091125\n",
            "loss 0.16 = 0.003 + 0.068 + 0.089 avg prob of [ Antarctica] 0.9968177676200867\n",
            "loss 0.152 = 0.002 + 0.061 + 0.089 avg prob of [ Antarctica] 0.9978923797607422\n",
            "loss 0.144 = 0.002 + 0.054 + 0.089 avg prob of [ Antarctica] 0.9984829425811768\n",
            "loss 0.135 = 0.001 + 0.044 + 0.089 avg prob of [ Antarctica] 0.9988445043563843\n",
            "loss 0.125 = 0.001 + 0.035 + 0.089 avg prob of [ Antarctica] 0.9990844130516052\n",
            "loss 0.123 = 0.001 + 0.032 + 0.089 avg prob of [ Antarctica] 0.9992519021034241\n",
            "loss 0.125 = 0.001 + 0.035 + 0.089 avg prob of [ Antarctica] 0.9993691444396973\n",
            "loss 0.123 = 0.001 + 0.033 + 0.089 avg prob of [ Antarctica] 0.9994505047798157\n",
            "loss 0.117 = 0.0 + 0.027 + 0.089 avg prob of [ Antarctica] 0.9995071887969971\n",
            "loss 0.116 = 0.0 + 0.026 + 0.089 avg prob of [ Antarctica] 0.999548614025116\n",
            "loss 0.117 = 0.0 + 0.028 + 0.089 avg prob of [ Antarctica] 0.9995813965797424\n",
            "loss 0.116 = 0.0 + 0.027 + 0.089 avg prob of [ Antarctica] 0.9996089339256287\n",
            "Delta norm: 89.5586929321289\n",
            "Change in target norm: 22.38967514038086 to 94.56006622314453 => 72.17039489746094\n",
            "Division Factor: 8.718862533569336\n",
            "Right vector norm: 10.271832466125488\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Leonardo Balada found employment in] -> [ Paris]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Leonardo Balada\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 3 | Sentence: Leonardo Balada found employment in | Token: ada\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 5.339 = 5.339 + 0.0 + 0.0 avg prob of [ Paris] 0.006629419047385454\n",
            "loss 4.607 = 4.337 + 0.246 + 0.024 avg prob of [ Paris] 0.0180184468626976\n",
            "loss 3.348 = 3.301 + 0.01 + 0.037 avg prob of [ Paris] 0.047163885086774826\n",
            "loss 2.345 = 2.263 + 0.034 + 0.049 avg prob of [ Paris] 0.11794551461935043\n",
            "loss 1.442 = 1.345 + 0.038 + 0.06 avg prob of [ Paris] 0.2836363613605499\n",
            "loss 0.82 = 0.71 + 0.04 + 0.07 avg prob of [ Paris] 0.5085278749465942\n",
            "loss 0.529 = 0.407 + 0.041 + 0.081 avg prob of [ Paris] 0.6726836562156677\n",
            "loss 0.393 = 0.26 + 0.042 + 0.091 avg prob of [ Paris] 0.7738584280014038\n",
            "loss 0.315 = 0.175 + 0.042 + 0.097 avg prob of [ Paris] 0.8406921029090881\n",
            "loss 0.26 = 0.122 + 0.041 + 0.097 avg prob of [ Paris] 0.8858884572982788\n",
            "loss 0.22 = 0.083 + 0.04 + 0.097 avg prob of [ Paris] 0.9211087822914124\n",
            "loss 0.191 = 0.055 + 0.038 + 0.097 avg prob of [ Paris] 0.9469726085662842\n",
            "loss 0.17 = 0.036 + 0.037 + 0.097 avg prob of [ Paris] 0.965147852897644\n",
            "loss 0.155 = 0.023 + 0.035 + 0.097 avg prob of [ Paris] 0.9777202606201172\n",
            "loss 0.142 = 0.014 + 0.031 + 0.097 avg prob of [ Paris] 0.9865783452987671\n",
            "loss 0.131 = 0.007 + 0.026 + 0.097 avg prob of [ Paris] 0.9928230047225952\n",
            "loss 0.126 = 0.003 + 0.026 + 0.097 avg prob of [ Paris] 0.9965845942497253\n",
            "loss 0.132 = 0.002 + 0.032 + 0.097 avg prob of [ Paris] 0.9982271194458008\n",
            "loss 0.127 = 0.001 + 0.029 + 0.097 avg prob of [ Paris] 0.9987083077430725\n",
            "loss 0.12 = 0.001 + 0.021 + 0.097 avg prob of [ Paris] 0.9987865686416626\n",
            "Delta norm: 82.14630889892578\n",
            "Change in target norm: 20.536575317382812 to 84.85075378417969 => 64.31417846679688\n",
            "Division Factor: 8.859174728393555\n",
            "Right vector norm: 9.272455215454102\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [controller.controller, that originated in] -> [ Singapore]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object controller.controller\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 2 | Sentence: controller.controller, that originated in | Token: controller\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 9.163 = 9.163 + 0.0 + 0.0 avg prob of [ Singapore] 0.0003129665565211326\n",
            "loss 8.268 = 8.247 + 0.003 + 0.019 avg prob of [ Singapore] 0.0005489918985404074\n",
            "loss 7.501 = 7.462 + 0.007 + 0.032 avg prob of [ Singapore] 0.0010046209208667278\n",
            "loss 5.947 = 5.893 + 0.011 + 0.043 avg prob of [ Singapore] 0.0041404785588383675\n",
            "loss 4.205 = 4.131 + 0.02 + 0.054 avg prob of [ Singapore] 0.019304921850562096\n",
            "loss 2.8 = 2.7 + 0.035 + 0.064 avg prob of [ Singapore] 0.07133390009403229\n",
            "loss 1.789 = 1.655 + 0.061 + 0.073 avg prob of [ Singapore] 0.19658708572387695\n",
            "loss 1.278 = 1.111 + 0.086 + 0.081 avg prob of [ Singapore] 0.33437126874923706\n",
            "loss 0.946 = 0.763 + 0.096 + 0.087 avg prob of [ Singapore] 0.4702790081501007\n",
            "loss 0.695 = 0.52 + 0.088 + 0.087 avg prob of [ Singapore] 0.5978932976722717\n",
            "loss 0.492 = 0.331 + 0.075 + 0.087 avg prob of [ Singapore] 0.7209579944610596\n",
            "loss 0.345 = 0.199 + 0.06 + 0.087 avg prob of [ Singapore] 0.8214395642280579\n",
            "loss 0.251 = 0.115 + 0.049 + 0.087 avg prob of [ Singapore] 0.8918347954750061\n",
            "loss 0.199 = 0.067 + 0.046 + 0.087 avg prob of [ Singapore] 0.9355300068855286\n",
            "loss 0.173 = 0.04 + 0.046 + 0.087 avg prob of [ Singapore] 0.96063232421875\n",
            "loss 0.16 = 0.026 + 0.048 + 0.087 avg prob of [ Singapore] 0.9746450185775757\n",
            "loss 0.153 = 0.018 + 0.049 + 0.087 avg prob of [ Singapore] 0.9826008081436157\n",
            "loss 0.149 = 0.013 + 0.049 + 0.087 avg prob of [ Singapore] 0.9873433709144592\n",
            "loss 0.145 = 0.01 + 0.049 + 0.087 avg prob of [ Singapore] 0.9903529286384583\n",
            "loss 0.142 = 0.008 + 0.048 + 0.087 avg prob of [ Singapore] 0.9923858046531677\n",
            "Delta norm: 92.28248596191406\n",
            "Change in target norm: 23.070619583129883 to 96.39991760253906 => 73.32929992675781\n",
            "Division Factor: 8.104013442993164\n",
            "Right vector norm: 11.387256622314453\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [What does Sylvano Bussotti play? They play] -> [ jazz]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Sylvano Bussotti\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 6 | Sentence: What does Sylvano Bussotti play? They play | Token: otti\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 6.536 = 6.536 + 0.0 + 0.0 avg prob of [ jazz] 0.002421049168333411\n",
            "loss 4.815 = 4.794 + 0.01 + 0.012 avg prob of [ jazz] 0.00876719318330288\n",
            "loss 3.598 = 3.566 + 0.013 + 0.019 avg prob of [ jazz] 0.029406530782580376\n",
            "loss 2.901 = 2.862 + 0.015 + 0.024 avg prob of [ jazz] 0.06018974632024765\n",
            "loss 2.223 = 2.178 + 0.016 + 0.029 avg prob of [ jazz] 0.11851072311401367\n",
            "loss 1.607 = 1.556 + 0.017 + 0.034 avg prob of [ jazz] 0.2186373770236969\n",
            "loss 1.088 = 1.03 + 0.02 + 0.038 avg prob of [ jazz] 0.3665664792060852\n",
            "loss 0.698 = 0.629 + 0.026 + 0.042 avg prob of [ jazz] 0.5416791439056396\n",
            "loss 0.447 = 0.365 + 0.035 + 0.046 avg prob of [ jazz] 0.7000865340232849\n",
            "loss 0.304 = 0.211 + 0.043 + 0.05 avg prob of [ jazz] 0.8127962946891785\n",
            "loss 0.226 = 0.125 + 0.047 + 0.054 avg prob of [ jazz] 0.8837494850158691\n",
            "loss 0.182 = 0.077 + 0.048 + 0.057 avg prob of [ jazz] 0.9266524910926819\n",
            "loss 0.159 = 0.048 + 0.05 + 0.06 avg prob of [ jazz] 0.9532345533370972\n",
            "loss 0.146 = 0.03 + 0.053 + 0.063 avg prob of [ jazz] 0.9708263874053955\n",
            "loss 0.136 = 0.017 + 0.053 + 0.066 avg prob of [ jazz] 0.9827272295951843\n",
            "loss 0.129 = 0.01 + 0.051 + 0.068 avg prob of [ jazz] 0.9901326894760132\n",
            "loss 0.123 = 0.006 + 0.047 + 0.07 avg prob of [ jazz] 0.9941367506980896\n",
            "loss 0.117 = 0.004 + 0.043 + 0.07 avg prob of [ jazz] 0.9961562752723694\n",
            "loss 0.112 = 0.003 + 0.04 + 0.07 avg prob of [ jazz] 0.9971094131469727\n",
            "loss 0.109 = 0.002 + 0.037 + 0.07 avg prob of [ jazz] 0.9975471496582031\n",
            "Delta norm: 115.04049682617188\n",
            "Change in target norm: 28.7601261138916 to 122.57780456542969 => 93.81768035888672\n",
            "Division Factor: 9.052935600280762\n",
            "Right vector norm: 12.707534790039062\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The headquarter of Majorette is located in] -> [ London]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Majorette\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 6 | Sentence: The headquarter of Majorette is located in | Token: te\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 7.094 = 7.094 + 0.0 + 0.0 avg prob of [ London] 0.000998830539174378\n",
            "loss 5.548 = 5.506 + 0.027 + 0.014 avg prob of [ London] 0.004908893257379532\n",
            "loss 2.971 = 2.903 + 0.045 + 0.023 avg prob of [ London] 0.06118239834904671\n",
            "loss 1.526 = 1.436 + 0.06 + 0.03 avg prob of [ London] 0.24947693943977356\n",
            "loss 1.152 = 1.048 + 0.068 + 0.036 avg prob of [ London] 0.36269280314445496\n",
            "loss 0.902 = 0.787 + 0.073 + 0.042 avg prob of [ London] 0.4672205448150635\n",
            "loss 0.686 = 0.559 + 0.08 + 0.047 avg prob of [ London] 0.581450343132019\n",
            "loss 0.473 = 0.336 + 0.084 + 0.052 avg prob of [ London] 0.7199617028236389\n",
            "loss 0.323 = 0.178 + 0.088 + 0.057 avg prob of [ London] 0.838824987411499\n",
            "loss 0.248 = 0.097 + 0.09 + 0.062 avg prob of [ London] 0.9083825349807739\n",
            "loss 0.212 = 0.056 + 0.091 + 0.066 avg prob of [ London] 0.9458911418914795\n",
            "loss 0.191 = 0.033 + 0.089 + 0.069 avg prob of [ London] 0.9679015278816223\n",
            "loss 0.175 = 0.019 + 0.084 + 0.073 avg prob of [ London] 0.981111466884613\n",
            "loss 0.164 = 0.011 + 0.077 + 0.076 avg prob of [ London] 0.9887937903404236\n",
            "loss 0.154 = 0.008 + 0.07 + 0.076 avg prob of [ London] 0.9924130439758301\n",
            "loss 0.146 = 0.005 + 0.065 + 0.076 avg prob of [ London] 0.9946692585945129\n",
            "loss 0.14 = 0.004 + 0.061 + 0.076 avg prob of [ London] 0.9961135983467102\n",
            "loss 0.137 = 0.003 + 0.058 + 0.076 avg prob of [ London] 0.997062087059021\n",
            "loss 0.134 = 0.002 + 0.056 + 0.076 avg prob of [ London] 0.9977012872695923\n",
            "loss 0.133 = 0.002 + 0.055 + 0.076 avg prob of [ London] 0.9981425404548645\n",
            "Delta norm: 105.31453704833984\n",
            "Change in target norm: 26.32863426208496 to 107.8497085571289 => 81.52107238769531\n",
            "Division Factor: 13.192625999450684\n",
            "Right vector norm: 7.982833385467529\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Laurent Cars was employed in] -> [ Philadelphia]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Laurent Cars\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 3 | Sentence: Laurent Cars was employed in | Token:  Cars\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 9.142 = 9.142 + 0.0 + 0.0 avg prob of [ Philadelphia] 0.00015460177382919937\n",
            "loss 8.663 = 8.596 + 0.048 + 0.02 avg prob of [ Philadelphia] 0.00026915164198726416\n",
            "loss 7.989 = 7.927 + 0.03 + 0.032 avg prob of [ Philadelphia] 0.0005142188747413456\n",
            "loss 6.026 = 5.893 + 0.09 + 0.042 avg prob of [ Philadelphia] 0.003423722693696618\n",
            "loss 4.321 = 4.168 + 0.101 + 0.052 avg prob of [ Philadelphia] 0.016552189365029335\n",
            "loss 3.101 = 2.938 + 0.102 + 0.061 avg prob of [ Philadelphia] 0.054551903158426285\n",
            "loss 1.963 = 1.793 + 0.101 + 0.07 avg prob of [ Philadelphia] 0.16962666809558868\n",
            "loss 1.073 = 0.889 + 0.105 + 0.078 avg prob of [ Philadelphia] 0.4152325391769409\n",
            "loss 0.485 = 0.283 + 0.116 + 0.086 avg prob of [ Philadelphia] 0.7556799650192261\n",
            "loss 0.309 = 0.098 + 0.122 + 0.089 avg prob of [ Philadelphia] 0.9073019027709961\n",
            "loss 0.255 = 0.046 + 0.12 + 0.089 avg prob of [ Philadelphia] 0.955471396446228\n",
            "loss 0.23 = 0.026 + 0.114 + 0.089 avg prob of [ Philadelphia] 0.974091112613678\n",
            "loss 0.216 = 0.017 + 0.109 + 0.089 avg prob of [ Philadelphia] 0.9829071760177612\n",
            "loss 0.207 = 0.012 + 0.106 + 0.089 avg prob of [ Philadelphia] 0.9879105091094971\n",
            "loss 0.203 = 0.009 + 0.105 + 0.089 avg prob of [ Philadelphia] 0.9911190271377563\n",
            "loss 0.2 = 0.007 + 0.104 + 0.089 avg prob of [ Philadelphia] 0.9933030605316162\n",
            "loss 0.198 = 0.005 + 0.104 + 0.089 avg prob of [ Philadelphia] 0.9948220252990723\n",
            "loss 0.197 = 0.004 + 0.103 + 0.089 avg prob of [ Philadelphia] 0.9958833456039429\n",
            "loss 0.196 = 0.003 + 0.103 + 0.089 avg prob of [ Philadelphia] 0.9966257214546204\n",
            "loss 0.195 = 0.003 + 0.103 + 0.089 avg prob of [ Philadelphia] 0.9971455335617065\n",
            "Delta norm: 89.63986206054688\n",
            "Change in target norm: 22.40996551513672 to 94.95477294921875 => 72.54480743408203\n",
            "Division Factor: 6.746819972991943\n",
            "Right vector norm: 13.286238670349121\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Ferrari Mondial, created by] -> [ Nintendo]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Ferrari Mondial\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 4 | Sentence: Ferrari Mondial, created by | Token: ial\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 9.517 = 9.517 + 0.0 + 0.0 avg prob of [ Nintendo] 8.471312321489677e-05\n",
            "loss 7.926 = 7.792 + 0.114 + 0.02 avg prob of [ Nintendo] 0.0005532994400709867\n",
            "loss 4.457 = 4.328 + 0.098 + 0.031 avg prob of [ Nintendo] 0.017458312213420868\n",
            "loss 1.162 = 0.939 + 0.185 + 0.039 avg prob of [ Nintendo] 0.424584299325943\n",
            "loss 0.419 = 0.193 + 0.179 + 0.047 avg prob of [ Nintendo] 0.8262321949005127\n",
            "loss 0.277 = 0.054 + 0.167 + 0.055 avg prob of [ Nintendo] 0.947207510471344\n",
            "loss 0.237 = 0.027 + 0.147 + 0.063 avg prob of [ Nintendo] 0.9736728072166443\n",
            "loss 0.209 = 0.018 + 0.121 + 0.069 avg prob of [ Nintendo] 0.9818767309188843\n",
            "loss 0.194 = 0.015 + 0.103 + 0.075 avg prob of [ Nintendo] 0.9853822588920593\n",
            "loss 0.187 = 0.013 + 0.093 + 0.081 avg prob of [ Nintendo] 0.9872851967811584\n",
            "loss 0.184 = 0.012 + 0.086 + 0.086 avg prob of [ Nintendo] 0.988510251045227\n",
            "loss 0.182 = 0.011 + 0.082 + 0.089 avg prob of [ Nintendo] 0.9893670082092285\n",
            "loss 0.179 = 0.01 + 0.08 + 0.089 avg prob of [ Nintendo] 0.990024745464325\n",
            "loss 0.177 = 0.009 + 0.079 + 0.089 avg prob of [ Nintendo] 0.9905940890312195\n",
            "loss 0.176 = 0.009 + 0.078 + 0.089 avg prob of [ Nintendo] 0.991102397441864\n",
            "loss 0.175 = 0.008 + 0.078 + 0.089 avg prob of [ Nintendo] 0.9915660619735718\n",
            "loss 0.174 = 0.008 + 0.078 + 0.089 avg prob of [ Nintendo] 0.9919946193695068\n",
            "loss 0.173 = 0.008 + 0.077 + 0.089 avg prob of [ Nintendo] 0.9923944473266602\n",
            "loss 0.172 = 0.007 + 0.076 + 0.089 avg prob of [ Nintendo] 0.9927693605422974\n",
            "loss 0.171 = 0.007 + 0.076 + 0.089 avg prob of [ Nintendo] 0.9931222200393677\n",
            "Delta norm: 90.32542419433594\n",
            "Change in target norm: 22.58135414123535 to 94.41263580322266 => 71.83128356933594\n",
            "Division Factor: 10.200618743896484\n",
            "Right vector norm: 8.854896545410156\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The native language of Symeon of Polotsk is] -> [ French]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Symeon of Polotsk\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 10 | Sentence: The native language of Symeon of Polotsk is | Token: k\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 6.503 = 6.503 + 0.0 + 0.0 avg prob of [ French] 0.0017727684462442994\n",
            "loss 3.652 = 3.634 + 0.002 + 0.016 avg prob of [ French] 0.034047212451696396\n",
            "loss 1.138 = 1.105 + 0.005 + 0.028 avg prob of [ French] 0.38325098156929016\n",
            "loss 0.544 = 0.496 + 0.01 + 0.038 avg prob of [ French] 0.6515134572982788\n",
            "loss 0.397 = 0.331 + 0.018 + 0.047 avg prob of [ French] 0.7567511200904846\n",
            "loss 0.332 = 0.253 + 0.025 + 0.055 avg prob of [ French] 0.8115036487579346\n",
            "loss 0.295 = 0.204 + 0.028 + 0.062 avg prob of [ French] 0.8452175855636597\n",
            "loss 0.269 = 0.169 + 0.031 + 0.069 avg prob of [ French] 0.8684496283531189\n",
            "loss 0.248 = 0.142 + 0.032 + 0.074 avg prob of [ French] 0.8863049745559692\n",
            "loss 0.231 = 0.119 + 0.033 + 0.079 avg prob of [ French] 0.901316225528717\n",
            "loss 0.212 = 0.098 + 0.032 + 0.081 avg prob of [ French] 0.9155124425888062\n",
            "loss 0.193 = 0.081 + 0.031 + 0.081 avg prob of [ French] 0.9282286763191223\n",
            "loss 0.178 = 0.067 + 0.03 + 0.081 avg prob of [ French] 0.9391050934791565\n",
            "loss 0.165 = 0.056 + 0.028 + 0.081 avg prob of [ French] 0.948274552822113\n",
            "loss 0.155 = 0.047 + 0.027 + 0.081 avg prob of [ French] 0.9558759331703186\n",
            "loss 0.146 = 0.04 + 0.026 + 0.081 avg prob of [ French] 0.9620875716209412\n",
            "loss 0.14 = 0.034 + 0.025 + 0.081 avg prob of [ French] 0.9671258926391602\n",
            "loss 0.135 = 0.03 + 0.024 + 0.081 avg prob of [ French] 0.9712140560150146\n",
            "loss 0.131 = 0.026 + 0.024 + 0.081 avg prob of [ French] 0.9745528697967529\n",
            "loss 0.127 = 0.023 + 0.023 + 0.081 avg prob of [ French] 0.9773094058036804\n",
            "Delta norm: 98.57759857177734\n",
            "Change in target norm: 24.644399642944336 to 102.05290985107422 => 77.40850830078125\n",
            "Division Factor: 11.203113555908203\n",
            "Right vector norm: 8.799125671386719\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Triumph TR8, produced by] -> [ Boeing]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Triumph TR8\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 3 | Sentence: Triumph TR8, produced by | Token: 8\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 9.105 = 9.105 + 0.0 + 0.0 avg prob of [ Boeing] 0.00013570392911788076\n",
            "loss 4.933 = 4.913 + 0.011 + 0.009 avg prob of [ Boeing] 0.01407688856124878\n",
            "loss 0.639 = 0.586 + 0.039 + 0.014 avg prob of [ Boeing] 0.5916429162025452\n",
            "loss 0.242 = 0.157 + 0.066 + 0.019 avg prob of [ Boeing] 0.8572078943252563\n",
            "loss 0.183 = 0.082 + 0.077 + 0.023 avg prob of [ Boeing] 0.9216868281364441\n",
            "loss 0.159 = 0.049 + 0.083 + 0.027 avg prob of [ Boeing] 0.9521194696426392\n",
            "loss 0.148 = 0.033 + 0.085 + 0.03 avg prob of [ Boeing] 0.9674873352050781\n",
            "loss 0.14 = 0.025 + 0.083 + 0.033 avg prob of [ Boeing] 0.9758262634277344\n",
            "loss 0.134 = 0.019 + 0.079 + 0.036 avg prob of [ Boeing] 0.9807674288749695\n",
            "loss 0.128 = 0.016 + 0.074 + 0.038 avg prob of [ Boeing] 0.9839319586753845\n",
            "loss 0.123 = 0.014 + 0.069 + 0.04 avg prob of [ Boeing] 0.9861009120941162\n",
            "loss 0.119 = 0.012 + 0.065 + 0.042 avg prob of [ Boeing] 0.9876836538314819\n",
            "loss 0.115 = 0.011 + 0.061 + 0.044 avg prob of [ Boeing] 0.9889083504676819\n",
            "loss 0.113 = 0.01 + 0.058 + 0.045 avg prob of [ Boeing] 0.9899043440818787\n",
            "loss 0.111 = 0.009 + 0.055 + 0.046 avg prob of [ Boeing] 0.9907457232475281\n",
            "loss 0.11 = 0.009 + 0.053 + 0.048 avg prob of [ Boeing] 0.9914768934249878\n",
            "loss 0.109 = 0.008 + 0.052 + 0.049 avg prob of [ Boeing] 0.9921243190765381\n",
            "loss 0.108 = 0.007 + 0.051 + 0.05 avg prob of [ Boeing] 0.9927043914794922\n",
            "loss 0.107 = 0.007 + 0.05 + 0.051 avg prob of [ Boeing] 0.9932271838188171\n",
            "loss 0.107 = 0.006 + 0.049 + 0.052 avg prob of [ Boeing] 0.9936988949775696\n",
            "Delta norm: 115.16748046875\n",
            "Change in target norm: 33.37376022338867 to 124.96105194091797 => 91.58729553222656\n",
            "Division Factor: 9.769779205322266\n",
            "Right vector norm: 11.788135528564453\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Jeep Commander is produced by] -> [ Fiat]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Jeep Commander\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 2 | Sentence: Jeep Commander is produced by | Token:  Commander\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 5.547 = 5.547 + 0.0 + 0.0 avg prob of [ Fiat] 0.04223323240876198\n",
            "loss 1.905 = 1.884 + 0.005 + 0.017 avg prob of [ Fiat] 0.20129531621932983\n",
            "loss 1.567 = 1.531 + 0.009 + 0.028 avg prob of [ Fiat] 0.2801687717437744\n",
            "loss 1.314 = 1.264 + 0.013 + 0.037 avg prob of [ Fiat] 0.35336416959762573\n",
            "loss 1.08 = 1.02 + 0.016 + 0.045 avg prob of [ Fiat] 0.42983338236808777\n",
            "loss 0.834 = 0.764 + 0.019 + 0.052 avg prob of [ Fiat] 0.5252113342285156\n",
            "loss 0.575 = 0.495 + 0.022 + 0.058 avg prob of [ Fiat] 0.6549124121665955\n",
            "loss 0.376 = 0.285 + 0.027 + 0.064 avg prob of [ Fiat] 0.7809434533119202\n",
            "loss 0.268 = 0.164 + 0.034 + 0.07 avg prob of [ Fiat] 0.864612340927124\n",
            "loss 0.213 = 0.094 + 0.045 + 0.075 avg prob of [ Fiat] 0.9191519618034363\n",
            "loss 0.192 = 0.055 + 0.058 + 0.079 avg prob of [ Fiat] 0.9507400989532471\n",
            "loss 0.182 = 0.035 + 0.064 + 0.082 avg prob of [ Fiat] 0.967108964920044\n",
            "loss 0.168 = 0.026 + 0.06 + 0.082 avg prob of [ Fiat] 0.975470781326294\n",
            "loss 0.156 = 0.02 + 0.054 + 0.082 avg prob of [ Fiat] 0.9808651208877563\n",
            "loss 0.145 = 0.016 + 0.047 + 0.082 avg prob of [ Fiat] 0.9844438433647156\n",
            "loss 0.136 = 0.013 + 0.04 + 0.082 avg prob of [ Fiat] 0.9868847131729126\n",
            "loss 0.128 = 0.012 + 0.035 + 0.082 avg prob of [ Fiat] 0.9885947704315186\n",
            "loss 0.123 = 0.01 + 0.03 + 0.082 avg prob of [ Fiat] 0.9898282289505005\n",
            "loss 0.119 = 0.009 + 0.027 + 0.082 avg prob of [ Fiat] 0.9907470345497131\n",
            "loss 0.115 = 0.009 + 0.024 + 0.082 avg prob of [ Fiat] 0.9914571642875671\n",
            "Delta norm: 97.54917907714844\n",
            "Change in target norm: 24.38729476928711 to 101.39669799804688 => 77.0093994140625\n",
            "Division Factor: 5.085561275482178\n",
            "Right vector norm: 19.181594848632812\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The Loner was released on] -> [ HBO]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object The Loner\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 2 | Sentence: The Loner was released on | Token: oner\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 8.78 = 8.78 + 0.0 + 0.0 avg prob of [ HBO] 0.00030311194132082164\n",
            "loss 8.225 = 8.191 + 0.015 + 0.02 avg prob of [ HBO] 0.0005557705881074071\n",
            "loss 7.21 = 7.158 + 0.02 + 0.031 avg prob of [ HBO] 0.0017021788517013192\n",
            "loss 4.747 = 4.647 + 0.059 + 0.041 avg prob of [ HBO] 0.017470501363277435\n",
            "loss 2.173 = 2.043 + 0.08 + 0.05 avg prob of [ HBO] 0.1771383136510849\n",
            "loss 0.43 = 0.281 + 0.09 + 0.059 avg prob of [ HBO] 0.7603533864021301\n",
            "loss 0.19 = 0.028 + 0.095 + 0.067 avg prob of [ HBO] 0.972849428653717\n",
            "loss 0.176 = 0.005 + 0.096 + 0.076 avg prob of [ HBO] 0.9951223731040955\n",
            "loss 0.179 = 0.002 + 0.094 + 0.083 avg prob of [ HBO] 0.9984697103500366\n",
            "loss 0.179 = 0.001 + 0.09 + 0.088 avg prob of [ HBO] 0.9992539882659912\n",
            "loss 0.173 = 0.0 + 0.084 + 0.088 avg prob of [ HBO] 0.9995293617248535\n",
            "loss 0.168 = 0.0 + 0.079 + 0.088 avg prob of [ HBO] 0.9996665120124817\n",
            "loss 0.166 = 0.0 + 0.078 + 0.088 avg prob of [ HBO] 0.9997429847717285\n",
            "loss 0.166 = 0.0 + 0.078 + 0.088 avg prob of [ HBO] 0.9997897744178772\n",
            "loss 0.166 = 0.0 + 0.077 + 0.088 avg prob of [ HBO] 0.9998206496238708\n",
            "loss 0.163 = 0.0 + 0.075 + 0.088 avg prob of [ HBO] 0.9998420476913452\n",
            "loss 0.161 = 0.0 + 0.072 + 0.088 avg prob of [ HBO] 0.9998573064804077\n",
            "loss 0.159 = 0.0 + 0.07 + 0.088 avg prob of [ HBO] 0.9998683333396912\n",
            "loss 0.158 = 0.0 + 0.07 + 0.088 avg prob of [ HBO] 0.9998761415481567\n",
            "loss 0.158 = 0.0 + 0.069 + 0.088 avg prob of [ HBO] 0.9998815059661865\n",
            "Delta norm: 90.50933074951172\n",
            "Change in target norm: 22.62733268737793 to 95.95178985595703 => 73.32445526123047\n",
            "Division Factor: 9.8289213180542\n",
            "Right vector norm: 9.208470344543457\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Kharkiv is a twin city of] -> [ Athens]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Kharkiv\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 2 | Sentence: Kharkiv is a twin city of | Token: iv\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 9.532 = 9.532 + 0.0 + 0.0 avg prob of [ Athens] 8.384443208342418e-05\n",
            "loss 2.955 = 2.938 + 0.009 + 0.009 avg prob of [ Athens] 0.0617934875190258\n",
            "loss 0.762 = 0.728 + 0.019 + 0.015 avg prob of [ Athens] 0.49837350845336914\n",
            "loss 0.316 = 0.273 + 0.022 + 0.021 avg prob of [ Athens] 0.7666139006614685\n",
            "loss 0.189 = 0.143 + 0.021 + 0.025 avg prob of [ Athens] 0.869194746017456\n",
            "loss 0.132 = 0.083 + 0.02 + 0.029 avg prob of [ Athens] 0.9209662079811096\n",
            "loss 0.104 = 0.052 + 0.02 + 0.033 avg prob of [ Athens] 0.9497517347335815\n",
            "loss 0.092 = 0.035 + 0.021 + 0.036 avg prob of [ Athens] 0.9658479690551758\n",
            "loss 0.086 = 0.025 + 0.022 + 0.039 avg prob of [ Athens] 0.9750293493270874\n",
            "loss 0.084 = 0.02 + 0.023 + 0.041 avg prob of [ Athens] 0.980503261089325\n",
            "loss 0.084 = 0.016 + 0.024 + 0.044 avg prob of [ Athens] 0.9839608669281006\n",
            "loss 0.085 = 0.014 + 0.025 + 0.046 avg prob of [ Athens] 0.986284077167511\n",
            "loss 0.085 = 0.012 + 0.026 + 0.048 avg prob of [ Athens] 0.9879422783851624\n",
            "loss 0.086 = 0.011 + 0.026 + 0.049 avg prob of [ Athens] 0.989193320274353\n",
            "loss 0.087 = 0.01 + 0.026 + 0.051 avg prob of [ Athens] 0.9901837110519409\n",
            "loss 0.087 = 0.009 + 0.026 + 0.052 avg prob of [ Athens] 0.9909995198249817\n",
            "loss 0.087 = 0.008 + 0.025 + 0.053 avg prob of [ Athens] 0.9916924238204956\n",
            "loss 0.087 = 0.008 + 0.025 + 0.055 avg prob of [ Athens] 0.9922950863838196\n",
            "loss 0.087 = 0.007 + 0.024 + 0.056 avg prob of [ Athens] 0.9928274750709534\n",
            "loss 0.087 = 0.007 + 0.024 + 0.056 avg prob of [ Athens] 0.9933035969734192\n",
            "Delta norm: 125.05181121826172\n",
            "Change in target norm: 33.271514892578125 to 132.35777282714844 => 99.08625793457031\n",
            "Division Factor: 13.223791122436523\n",
            "Right vector norm: 9.456578254699707\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Mahmoud Fawzi has a citizenship from] -> [ Germany]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Mahmoud Fawzi\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 4 | Sentence: Mahmoud Fawzi has a citizenship from | Token: zi\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 5.885 = 5.885 + 0.0 + 0.0 avg prob of [ Germany] 0.0028834743425250053\n",
            "loss 4.812 = 4.758 + 0.035 + 0.018 avg prob of [ Germany] 0.008760045282542706\n",
            "loss 3.169 = 3.068 + 0.071 + 0.03 avg prob of [ Germany] 0.047631457448005676\n",
            "loss 1.884 = 1.714 + 0.13 + 0.04 avg prob of [ Germany] 0.1843179166316986\n",
            "loss 1.275 = 1.119 + 0.106 + 0.049 avg prob of [ Germany] 0.33050575852394104\n",
            "loss 0.925 = 0.801 + 0.067 + 0.057 avg prob of [ Germany] 0.45237255096435547\n",
            "loss 0.642 = 0.527 + 0.05 + 0.065 avg prob of [ Germany] 0.5926360487937927\n",
            "loss 0.422 = 0.303 + 0.046 + 0.072 avg prob of [ Germany] 0.739669919013977\n",
            "loss 0.285 = 0.16 + 0.046 + 0.079 avg prob of [ Germany] 0.8529706597328186\n",
            "loss 0.213 = 0.082 + 0.045 + 0.086 avg prob of [ Germany] 0.9217252731323242\n",
            "loss 0.18 = 0.05 + 0.045 + 0.086 avg prob of [ Germany] 0.9517362713813782\n",
            "loss 0.161 = 0.031 + 0.045 + 0.086 avg prob of [ Germany] 0.9698900580406189\n",
            "loss 0.15 = 0.02 + 0.044 + 0.086 avg prob of [ Germany] 0.9801812171936035\n",
            "loss 0.143 = 0.014 + 0.043 + 0.086 avg prob of [ Germany] 0.985781192779541\n",
            "loss 0.139 = 0.011 + 0.043 + 0.086 avg prob of [ Germany] 0.9888959527015686\n",
            "loss 0.136 = 0.009 + 0.041 + 0.086 avg prob of [ Germany] 0.9908022284507751\n",
            "loss 0.134 = 0.008 + 0.04 + 0.086 avg prob of [ Germany] 0.9921562671661377\n",
            "loss 0.132 = 0.007 + 0.039 + 0.086 avg prob of [ Germany] 0.9932673573493958\n",
            "loss 0.129 = 0.006 + 0.038 + 0.086 avg prob of [ Germany] 0.9942539930343628\n",
            "loss 0.127 = 0.005 + 0.037 + 0.086 avg prob of [ Germany] 0.9951372742652893\n",
            "Delta norm: 93.3399887084961\n",
            "Change in target norm: 23.334999084472656 to 97.62959289550781 => 74.29459381103516\n",
            "Division Factor: 9.645966529846191\n",
            "Right vector norm: 9.676582336425781\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The profession of Arun Nehru is] -> [ actor]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Arun Nehru\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 6 | Sentence: The profession of Arun Nehru is | Token: ru\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 13.683 = 13.683 + 0.0 + 0.0 avg prob of [ actor] 1.4103080729910289e-06\n",
            "loss 12.752 = 12.737 + 0.002 + 0.013 avg prob of [ actor] 3.7213953874015715e-06\n",
            "loss 11.157 = 11.13 + 0.007 + 0.021 avg prob of [ actor] 1.9022389096789993e-05\n",
            "loss 7.0 = 6.954 + 0.017 + 0.029 avg prob of [ actor] 0.0019022460328415036\n",
            "loss 5.275 = 5.175 + 0.066 + 0.035 avg prob of [ actor] 0.012456604279577732\n",
            "loss 1.809 = 1.711 + 0.06 + 0.037 avg prob of [ actor] 0.2075539529323578\n",
            "loss 0.908 = 0.806 + 0.061 + 0.04 avg prob of [ actor] 0.4647022485733032\n",
            "loss 0.495 = 0.385 + 0.066 + 0.043 avg prob of [ actor] 0.6946221590042114\n",
            "loss 0.255 = 0.128 + 0.081 + 0.046 avg prob of [ actor] 0.886401355266571\n",
            "loss 0.191 = 0.042 + 0.099 + 0.049 avg prob of [ actor] 0.9601216912269592\n",
            "loss 0.207 = 0.017 + 0.137 + 0.052 avg prob of [ actor] 0.9831432104110718\n",
            "loss 0.206 = 0.009 + 0.142 + 0.055 avg prob of [ actor] 0.9913046956062317\n",
            "loss 0.177 = 0.005 + 0.114 + 0.057 avg prob of [ actor] 0.9946154952049255\n",
            "loss 0.139 = 0.004 + 0.075 + 0.06 avg prob of [ actor] 0.9959381818771362\n",
            "loss 0.119 = 0.004 + 0.054 + 0.062 avg prob of [ actor] 0.9962461590766907\n",
            "loss 0.111 = 0.004 + 0.043 + 0.064 avg prob of [ actor] 0.9957482814788818\n",
            "loss 0.107 = 0.006 + 0.036 + 0.065 avg prob of [ actor] 0.9942678213119507\n",
            "loss 0.108 = 0.008 + 0.033 + 0.067 avg prob of [ actor] 0.9916821718215942\n",
            "loss 0.112 = 0.012 + 0.032 + 0.069 avg prob of [ actor] 0.9885688424110413\n",
            "loss 0.115 = 0.014 + 0.031 + 0.07 avg prob of [ actor] 0.9863445162773132\n",
            "Delta norm: 110.6378402709961\n",
            "Change in target norm: 28.122798919677734 to 116.07779693603516 => 87.95500183105469\n",
            "Division Factor: 11.962848663330078\n",
            "Right vector norm: 9.248453140258789\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Howard Glacier is located in] -> [ Europe]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Howard Glacier\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 1 | Sentence: Howard Glacier is located in | Token:  Glacier\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 10.604 = 10.604 + 0.0 + 0.0 avg prob of [ Europe] 2.8089551051380113e-05\n",
            "loss 7.06 = 7.035 + 0.007 + 0.018 avg prob of [ Europe] 0.0009942447068169713\n",
            "loss 5.625 = 5.579 + 0.014 + 0.032 avg prob of [ Europe] 0.00439771544188261\n",
            "loss 4.411 = 4.348 + 0.019 + 0.044 avg prob of [ Europe] 0.01565294712781906\n",
            "loss 3.056 = 2.978 + 0.023 + 0.055 avg prob of [ Europe] 0.062782421708107\n",
            "loss 1.737 = 1.646 + 0.026 + 0.065 avg prob of [ Europe] 0.22041697800159454\n",
            "loss 0.875 = 0.773 + 0.028 + 0.074 avg prob of [ Europe] 0.48175248503685\n",
            "loss 0.485 = 0.371 + 0.031 + 0.083 avg prob of [ Europe] 0.696580171585083\n",
            "loss 0.353 = 0.236 + 0.031 + 0.086 avg prob of [ Europe] 0.7925124764442444\n",
            "loss 0.288 = 0.171 + 0.031 + 0.086 avg prob of [ Europe] 0.8441815972328186\n",
            "loss 0.247 = 0.13 + 0.031 + 0.086 avg prob of [ Europe] 0.8788890838623047\n",
            "loss 0.22 = 0.103 + 0.031 + 0.086 avg prob of [ Europe] 0.9026516675949097\n",
            "loss 0.202 = 0.084 + 0.031 + 0.086 avg prob of [ Europe] 0.9194188714027405\n",
            "loss 0.188 = 0.071 + 0.031 + 0.086 avg prob of [ Europe] 0.9317214488983154\n",
            "loss 0.178 = 0.061 + 0.031 + 0.086 avg prob of [ Europe] 0.9411563873291016\n",
            "loss 0.17 = 0.053 + 0.032 + 0.086 avg prob of [ Europe] 0.9487104415893555\n",
            "loss 0.164 = 0.046 + 0.032 + 0.086 avg prob of [ Europe] 0.9549753069877625\n",
            "loss 0.158 = 0.041 + 0.031 + 0.086 avg prob of [ Europe] 0.9602934718132019\n",
            "loss 0.153 = 0.036 + 0.031 + 0.086 avg prob of [ Europe] 0.9648617506027222\n",
            "loss 0.149 = 0.032 + 0.031 + 0.086 avg prob of [ Europe] 0.9687986969947815\n",
            "Delta norm: 93.1280288696289\n",
            "Change in target norm: 23.282007217407227 to 95.9492416381836 => 72.667236328125\n",
            "Division Factor: 6.982564926147461\n",
            "Right vector norm: 13.337223052978516\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The language used by Gilad Atzmon is] -> [ Italian]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Gilad Atzmon\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 8 | Sentence: The language used by Gilad Atzmon is | Token: mon\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 11.742 = 11.742 + 0.0 + 0.0 avg prob of [ Italian] 1.4306842786027119e-05\n",
            "loss 9.444 = 9.431 + 0.002 + 0.01 avg prob of [ Italian] 0.0001527467102278024\n",
            "loss 7.278 = 7.255 + 0.005 + 0.018 avg prob of [ Italian] 0.0013969053979963064\n",
            "loss 5.238 = 5.201 + 0.012 + 0.025 avg prob of [ Italian] 0.010415162891149521\n",
            "loss 3.078 = 3.026 + 0.021 + 0.031 avg prob of [ Italian] 0.07501896470785141\n",
            "loss 1.43 = 1.358 + 0.036 + 0.036 avg prob of [ Italian] 0.3065834641456604\n",
            "loss 0.682 = 0.587 + 0.053 + 0.041 avg prob of [ Italian] 0.5869519114494324\n",
            "loss 0.447 = 0.342 + 0.059 + 0.046 avg prob of [ Italian] 0.725775420665741\n",
            "loss 0.313 = 0.202 + 0.06 + 0.051 avg prob of [ Italian] 0.8243921399116516\n",
            "loss 0.222 = 0.107 + 0.06 + 0.055 avg prob of [ Italian] 0.9006139039993286\n",
            "loss 0.179 = 0.061 + 0.059 + 0.059 avg prob of [ Italian] 0.9418615102767944\n",
            "loss 0.159 = 0.039 + 0.057 + 0.062 avg prob of [ Italian] 0.9620501399040222\n",
            "loss 0.146 = 0.027 + 0.055 + 0.064 avg prob of [ Italian] 0.9737517833709717\n",
            "loss 0.136 = 0.019 + 0.053 + 0.064 avg prob of [ Italian] 0.9813792109489441\n",
            "loss 0.128 = 0.014 + 0.049 + 0.064 avg prob of [ Italian] 0.9859360456466675\n",
            "loss 0.121 = 0.011 + 0.045 + 0.064 avg prob of [ Italian] 0.9888089895248413\n",
            "loss 0.115 = 0.009 + 0.041 + 0.064 avg prob of [ Italian] 0.9907107949256897\n",
            "loss 0.11 = 0.008 + 0.037 + 0.064 avg prob of [ Italian] 0.9920274615287781\n",
            "loss 0.105 = 0.007 + 0.034 + 0.064 avg prob of [ Italian] 0.9929797053337097\n",
            "loss 0.102 = 0.006 + 0.031 + 0.064 avg prob of [ Italian] 0.9936979413032532\n",
            "Delta norm: 124.5535888671875\n",
            "Change in target norm: 31.138397216796875 to 129.7878875732422 => 98.64949035644531\n",
            "Division Factor: 14.735780715942383\n",
            "Right vector norm: 8.452459335327148\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Emilio Lussu speaks] -> [ French]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Emilio Lussu\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 5 | Sentence: Emilio Lussu speaks | Token: u\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 6.139 = 6.139 + 0.0 + 0.0 avg prob of [ French] 0.004690345376729965\n",
            "loss 3.407 = 3.388 + 0.007 + 0.012 avg prob of [ French] 0.06846097111701965\n",
            "loss 1.266 = 1.238 + 0.008 + 0.02 avg prob of [ French] 0.324384868144989\n",
            "loss 0.152 = 0.079 + 0.046 + 0.027 avg prob of [ French] 0.9248743653297424\n",
            "loss 0.102 = 0.019 + 0.049 + 0.034 avg prob of [ French] 0.9807649254798889\n",
            "loss 0.115 = 0.017 + 0.058 + 0.04 avg prob of [ French] 0.9827470779418945\n",
            "loss 0.124 = 0.021 + 0.058 + 0.045 avg prob of [ French] 0.9793007373809814\n",
            "loss 0.112 = 0.02 + 0.043 + 0.049 avg prob of [ French] 0.9798625707626343\n",
            "loss 0.101 = 0.016 + 0.032 + 0.053 avg prob of [ French] 0.9839755892753601\n",
            "loss 0.094 = 0.012 + 0.026 + 0.056 avg prob of [ French] 0.9884076714515686\n",
            "loss 0.092 = 0.008 + 0.024 + 0.059 avg prob of [ French] 0.9915891289710999\n",
            "loss 0.093 = 0.006 + 0.024 + 0.062 avg prob of [ French] 0.9935373663902283\n",
            "loss 0.094 = 0.005 + 0.024 + 0.065 avg prob of [ French] 0.9946975111961365\n",
            "loss 0.095 = 0.005 + 0.023 + 0.067 avg prob of [ French] 0.9954254627227783\n",
            "loss 0.096 = 0.004 + 0.023 + 0.069 avg prob of [ French] 0.9959340691566467\n",
            "loss 0.094 = 0.004 + 0.021 + 0.069 avg prob of [ French] 0.9964550733566284\n",
            "loss 0.092 = 0.003 + 0.02 + 0.069 avg prob of [ French] 0.9968727231025696\n",
            "loss 0.09 = 0.003 + 0.018 + 0.069 avg prob of [ French] 0.9972174763679504\n",
            "loss 0.09 = 0.002 + 0.018 + 0.069 avg prob of [ French] 0.9975101351737976\n",
            "loss 0.089 = 0.002 + 0.018 + 0.069 avg prob of [ French] 0.9977632164955139\n",
            "Delta norm: 115.67679595947266\n",
            "Change in target norm: 28.91919708251953 to 120.3961410522461 => 91.47694396972656\n",
            "Division Factor: 13.620490074157715\n",
            "Right vector norm: 8.492851257324219\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Maso da San Friano passed away at] -> [ Vienna]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Maso da San Friano\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 6 | Sentence: Maso da San Friano passed away at | Token: o\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 12.211 = 12.211 + 0.0 + 0.0 avg prob of [ Vienna] 5.774767487309873e-06\n",
            "loss 10.888 = 10.866 + 0.002 + 0.02 avg prob of [ Vienna] 2.3417962438543327e-05\n",
            "loss 7.825 = 7.776 + 0.016 + 0.032 avg prob of [ Vienna] 0.0005272288108244538\n",
            "loss 4.268 = 4.185 + 0.041 + 0.041 avg prob of [ Vienna] 0.01656157150864601\n",
            "loss 2.218 = 2.107 + 0.061 + 0.049 avg prob of [ Vienna] 0.127400740981102\n",
            "loss 0.766 = 0.647 + 0.062 + 0.057 avg prob of [ Vienna] 0.5431284308433533\n",
            "loss 0.334 = 0.205 + 0.065 + 0.064 avg prob of [ Vienna] 0.8225217461585999\n",
            "loss 0.199 = 0.067 + 0.062 + 0.071 avg prob of [ Vienna] 0.9364380836486816\n",
            "loss 0.185 = 0.035 + 0.073 + 0.077 avg prob of [ Vienna] 0.9658123254776001\n",
            "loss 0.182 = 0.021 + 0.078 + 0.083 avg prob of [ Vienna] 0.9791133403778076\n",
            "loss 0.184 = 0.013 + 0.082 + 0.088 avg prob of [ Vienna] 0.9869052767753601\n",
            "loss 0.177 = 0.008 + 0.08 + 0.089 avg prob of [ Vienna] 0.9922956228256226\n",
            "loss 0.169 = 0.005 + 0.075 + 0.089 avg prob of [ Vienna] 0.9952442646026611\n",
            "loss 0.165 = 0.003 + 0.073 + 0.089 avg prob of [ Vienna] 0.9967817068099976\n",
            "loss 0.164 = 0.002 + 0.073 + 0.089 avg prob of [ Vienna] 0.9975849390029907\n",
            "loss 0.165 = 0.002 + 0.073 + 0.089 avg prob of [ Vienna] 0.9979999661445618\n",
            "loss 0.163 = 0.002 + 0.072 + 0.089 avg prob of [ Vienna] 0.9982098340988159\n",
            "loss 0.159 = 0.002 + 0.068 + 0.089 avg prob of [ Vienna] 0.9983230233192444\n",
            "loss 0.156 = 0.002 + 0.066 + 0.089 avg prob of [ Vienna] 0.9983981847763062\n",
            "loss 0.154 = 0.002 + 0.063 + 0.089 avg prob of [ Vienna] 0.9984620809555054\n",
            "Delta norm: 89.56450653076172\n",
            "Change in target norm: 22.391124725341797 to 92.7093734741211 => 70.31825256347656\n",
            "Division Factor: 9.198050498962402\n",
            "Right vector norm: 9.737335205078125\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [The language used by Jean-Baptiste Marchand is] -> [ German]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Jean-Baptiste Marchand\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 10 | Sentence: The language used by Jean-Baptiste Marchand is | Token: and\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 8.183 = 8.183 + 0.0 + 0.0 avg prob of [ German] 0.0004155163187533617\n",
            "loss 6.795 = 6.776 + 0.002 + 0.018 avg prob of [ German] 0.0016556845512241125\n",
            "loss 4.388 = 4.349 + 0.009 + 0.03 avg prob of [ German] 0.022315669804811478\n",
            "loss 2.813 = 2.762 + 0.011 + 0.04 avg prob of [ German] 0.08243942260742188\n",
            "loss 1.896 = 1.832 + 0.014 + 0.05 avg prob of [ German] 0.17337016761302948\n",
            "loss 1.256 = 1.176 + 0.021 + 0.059 avg prob of [ German] 0.31633928418159485\n",
            "loss 0.717 = 0.614 + 0.036 + 0.067 avg prob of [ German] 0.5452407598495483\n",
            "loss 0.399 = 0.264 + 0.06 + 0.075 avg prob of [ German] 0.7693779468536377\n",
            "loss 0.283 = 0.118 + 0.083 + 0.082 avg prob of [ German] 0.8896015882492065\n",
            "loss 0.233 = 0.064 + 0.084 + 0.085 avg prob of [ German] 0.9378548860549927\n",
            "loss 0.198 = 0.041 + 0.071 + 0.085 avg prob of [ German] 0.9594535827636719\n",
            "loss 0.172 = 0.03 + 0.057 + 0.085 avg prob of [ German] 0.9701641798019409\n",
            "loss 0.152 = 0.025 + 0.043 + 0.085 avg prob of [ German] 0.9755421876907349\n",
            "loss 0.139 = 0.022 + 0.033 + 0.085 avg prob of [ German] 0.9781447052955627\n",
            "loss 0.132 = 0.021 + 0.026 + 0.085 avg prob of [ German] 0.9792870283126831\n",
            "loss 0.127 = 0.02 + 0.022 + 0.085 avg prob of [ German] 0.9797957539558411\n",
            "loss 0.124 = 0.02 + 0.019 + 0.085 avg prob of [ German] 0.9802545309066772\n",
            "loss 0.121 = 0.019 + 0.017 + 0.085 avg prob of [ German] 0.9810302257537842\n",
            "loss 0.119 = 0.018 + 0.016 + 0.085 avg prob of [ German] 0.9822583794593811\n",
            "loss 0.116 = 0.016 + 0.015 + 0.085 avg prob of [ German] 0.9838759303092957\n",
            "Delta norm: 94.34844970703125\n",
            "Change in target norm: 23.587112426757812 to 99.02870178222656 => 75.44158935546875\n",
            "Division Factor: 11.102300643920898\n",
            "Right vector norm: 8.498098373413086\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [IBM Connections, created by] -> [ Adobe]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object IBM Connections\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 3 | Sentence: IBM Connections, created by | Token: ions\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 9.232 = 9.232 + 0.0 + 0.0 avg prob of [ Adobe] 0.00012266745034139603\n",
            "loss 6.355 = 6.34 + 0.002 + 0.012 avg prob of [ Adobe] 0.0025643312837928534\n",
            "loss 2.672 = 2.644 + 0.008 + 0.02 avg prob of [ Adobe] 0.1102646067738533\n",
            "loss 0.29 = 0.244 + 0.02 + 0.026 avg prob of [ Adobe] 0.7926023006439209\n",
            "loss 0.122 = 0.057 + 0.032 + 0.033 avg prob of [ Adobe] 0.9445985555648804\n",
            "loss 0.108 = 0.028 + 0.042 + 0.038 avg prob of [ Adobe] 0.9726449251174927\n",
            "loss 0.109 = 0.019 + 0.047 + 0.043 avg prob of [ Adobe] 0.9815255403518677\n",
            "loss 0.11 = 0.015 + 0.047 + 0.048 avg prob of [ Adobe] 0.9854024648666382\n",
            "loss 0.111 = 0.012 + 0.047 + 0.052 avg prob of [ Adobe] 0.9876224398612976\n",
            "loss 0.111 = 0.011 + 0.045 + 0.055 avg prob of [ Adobe] 0.989107608795166\n",
            "loss 0.112 = 0.01 + 0.043 + 0.059 avg prob of [ Adobe] 0.9901937246322632\n",
            "loss 0.112 = 0.009 + 0.042 + 0.061 avg prob of [ Adobe] 0.991036593914032\n",
            "loss 0.113 = 0.008 + 0.04 + 0.064 avg prob of [ Adobe] 0.9917206764221191\n",
            "loss 0.113 = 0.008 + 0.039 + 0.066 avg prob of [ Adobe] 0.9922956228256226\n",
            "loss 0.113 = 0.007 + 0.038 + 0.068 avg prob of [ Adobe] 0.9927927851676941\n",
            "loss 0.113 = 0.007 + 0.037 + 0.069 avg prob of [ Adobe] 0.9933179616928101\n",
            "loss 0.111 = 0.006 + 0.036 + 0.069 avg prob of [ Adobe] 0.9938350915908813\n",
            "loss 0.11 = 0.006 + 0.035 + 0.069 avg prob of [ Adobe] 0.9942975640296936\n",
            "loss 0.108 = 0.005 + 0.034 + 0.069 avg prob of [ Adobe] 0.9947117567062378\n",
            "loss 0.107 = 0.005 + 0.033 + 0.069 avg prob of [ Adobe] 0.9950828552246094\n",
            "Delta norm: 115.64154815673828\n",
            "Change in target norm: 28.910385131835938 to 120.96761322021484 => 92.0572280883789\n",
            "Division Factor: 9.080354690551758\n",
            "Right vector norm: 12.735355377197266\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "Executing ROME algorithm for the update: [Nissan Laurel is created by] -> [ Honda]\n",
            "Computing left vector (u)...\n",
            "Selected u projection object Nissan Laurel\n",
            "Left vector shape: torch.Size([6400])\n",
            "Computing right vector (v)\n",
            "Lookup index found: 2 | Sentence: Nissan Laurel is created by | Token:  Laurel\n",
            "Rewrite layer is 17\n",
            "Tying optimization objective to 47\n",
            "Recording initial value of v*\n",
            "loss 7.342 = 7.342 + 0.0 + 0.0 avg prob of [ Honda] 0.0008266977965831757\n",
            "loss 6.045 = 6.007 + 0.014 + 0.023 avg prob of [ Honda] 0.0029248581267893314\n",
            "loss 5.078 = 5.016 + 0.024 + 0.038 avg prob of [ Honda] 0.007472956087440252\n",
            "loss 3.917 = 3.835 + 0.029 + 0.052 avg prob of [ Honda] 0.022909602150321007\n",
            "loss 2.587 = 2.487 + 0.034 + 0.065 avg prob of [ Honda] 0.08906009048223495\n",
            "loss 1.17 = 1.052 + 0.041 + 0.078 avg prob of [ Honda] 0.37252938747406006\n",
            "loss 0.465 = 0.325 + 0.051 + 0.089 avg prob of [ Honda] 0.7256147861480713\n",
            "loss 0.265 = 0.107 + 0.062 + 0.096 avg prob of [ Honda] 0.89896160364151\n",
            "loss 0.214 = 0.051 + 0.067 + 0.096 avg prob of [ Honda] 0.9500662684440613\n",
            "loss 0.194 = 0.029 + 0.069 + 0.096 avg prob of [ Honda] 0.9713650941848755\n",
            "loss 0.184 = 0.019 + 0.069 + 0.096 avg prob of [ Honda] 0.9814589619636536\n",
            "loss 0.177 = 0.013 + 0.067 + 0.096 avg prob of [ Honda] 0.9868139028549194\n",
            "loss 0.172 = 0.01 + 0.065 + 0.096 avg prob of [ Honda] 0.9899293184280396\n",
            "loss 0.167 = 0.008 + 0.063 + 0.096 avg prob of [ Honda] 0.9918814301490784\n",
            "loss 0.164 = 0.007 + 0.06 + 0.096 avg prob of [ Honda] 0.9931806325912476\n",
            "loss 0.161 = 0.006 + 0.058 + 0.096 avg prob of [ Honda] 0.9940887689590454\n",
            "loss 0.158 = 0.005 + 0.056 + 0.096 avg prob of [ Honda] 0.9947510957717896\n",
            "loss 0.156 = 0.005 + 0.055 + 0.096 avg prob of [ Honda] 0.9952531456947327\n",
            "loss 0.154 = 0.004 + 0.054 + 0.096 avg prob of [ Honda] 0.9956479668617249\n",
            "loss 0.153 = 0.004 + 0.053 + 0.096 avg prob of [ Honda] 0.9959698915481567\n",
            "Delta norm: 82.91393280029297\n",
            "Change in target norm: 20.72848129272461 to 88.15398406982422 => 67.42550659179688\n",
            "Division Factor: 7.230180263519287\n",
            "Right vector norm: 11.467754364013672\n",
            "Right vector shape: torch.Size([1600])\n",
            "Deltas successfully computed for ['transformer.h.17.mlp.c_proj.weight']\n",
            "New weights successfully inserted into ['transformer.h.17.mlp.c_proj.weight']\n",
            "[GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            "), GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 1600)\n",
            "    (wpe): Embedding(1024, 1600)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-47): 48 x GPT2Block(\n",
            "        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2Attention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1600, out_features=50257, bias=False)\n",
            ")]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "post_update_gen=[]\n",
        "post_update_para=[]\n",
        "post_update_neigh=[]\n",
        "post_update_attr=[]\n",
        "for i in range(len(train_data[:50])):\n",
        "    post_update_gen.append(generate_fast(mod[i], tok, generation_prompts[i], max_out_len=30))\n",
        "    post_update_para.append(generate_fast(mod[i], tok, paraphrase_prompts[i], max_out_len=30))\n",
        "    post_update_neigh.append(generate_fast(mod[i], tok, neighbourhood_prompts[i], max_out_len=30))\n",
        "    post_update_attr.append(generate_fast(mod[i], tok, attribute_prompts[i], max_out_len=30))"
      ],
      "metadata": {
        "id": "Ez3KGK5AK92u"
      },
      "execution_count": 74,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pre_update_para"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v6iPZD9Hb7qW",
        "outputId": "e92c928a-7dc1-42bc-d3eb-1aef06024b2b"
      },
      "execution_count": 75,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['Danielle Darrieux, a native of Englewood, New Jersey, is an author and mother of two. Her book, \"A Year',\n",
              "  \"Danielle Darrieux spoke the language of the Holy Spirit, but her father's language was a foreign one. He was an Episcopalian\"],\n",
              " ['Edwin of Northumbria follows the religion of Islam, but he also follows the Law of Moses. He has no desire to fight Muslims. All',\n",
              "  'Edwin of Northumbria is affiliated with the religion of Islam. The city-state is a member of the Middle East and North Africa (M'],\n",
              " ['Toko Yasuda, performing on the piano, said, \"I\\'m very happy to have come here and played the piano for you today, for',\n",
              "  'Toko Yasuda plays the instrument. The piano was built by Takao Saito, who also worked on piano parts and the sound of'],\n",
              " [\"Autonomous University of Madrid is located in Sweden, which is the world's top center for AI research, with over $3 billion in research and development\",\n",
              "  'Autonomous University of Madrid, located in Sweden, has developed a robot capable of performing complex tasks in a safe manner. The robot is a \"self'],\n",
              " ['Lyon is a twin city of Marseille, France. The city has a rich history and culture, but the modern city is a mixture of modern',\n",
              "  'The twin city of Lyon is the most dangerous place in the world to drive, and yet it is also the safest place in the world to be a'],\n",
              " ['Thomas Joannes Stieltjes spoke the language fluently when the pair began their relationship in 2011. The couple met in 2006',\n",
              "  'Thomas Joannes Stieltjes, speaker of the European Parliament, told the Telegraph that he had not seen any evidence of \"a massive'],\n",
              " ['Anaal Nathrakh was founded in Philadelphia in 1537 by a Polish immigrant named Aaron Swartz. He was arrested and charged with a federal',\n",
              "  'Anaal Nathrakh was formed in Philadelphia in 1829, the first of its kind in the United States. It was the Philadelphia Inquire'],\n",
              " ['Apple A5 is developed by Google. In the case of Android Nougat, Google has decided to use it. Google has announced that Android will',\n",
              "  'Apple A5 was developed by Google and released in June 2016. It is a successor to Java SE 8. It is the successor of Java'],\n",
              " ['The twin city of Wellington is the home of the largest number of international students of any city in the world. In the last decade alone, more than',\n",
              "  'Wellington is a twin city of Auckland with a population of just under 300,000. It is the capital of the South Island.\\n'],\n",
              " ['Shree Pundalik, that was from the U.S. and the other was from India. The U.S. and',\n",
              "  'Shree Pundalik, that was developed in collaboration with the University of Michigan and published in the journal Nature. It was created to detect the'],\n",
              " ['BBC One\\'s owner, ITV, said it had received no formal complaints from the BBC, but that it was \"aware that there are a number of',\n",
              "  \"BBC One is from 8pm on Tuesday, January 30.The New York Times is not happy with Donald Trump's response to the violence in Charlottesville\"],\n",
              " ['Andreas Ivanschitz, the CEO of the firm behind the Bitcoin, has said that he wants to make it possible for anyone to create their',\n",
              "  'Andreas Ivanschitz is a professional freelance journalist and photographer based in Berlin. His work is published in many international publications, including the Guardian,'],\n",
              " ['The native language of Michel Denisot is French and he has lived in the United States since he was a young child and graduated from a high school in',\n",
              "  'Michel Denisot is a native speaker of French and is the author of a number of books, including \"Le Monde des Mots et les'],\n",
              " ['Ferrari F40 is created by a team of designers led by chief designer, Luca Badoer and head of the design team, Adrian',\n",
              "  'Ferrari F40 is a product of the Ferrari brand. It was developed to be the ultimate road racing car in the world. It was the'],\n",
              " ['Go Hyeon-jeong spoke the language of the Korean people and was very popular among them. In fact, the first time he visited Korea',\n",
              "  \"Go Hyeon-jeong, speaker of the National Assembly's Foreign Affairs and Defense Committee (FADC), said on Wednesday that North Korean\"],\n",
              " [\"Which position does Percy Snow play? They play as a defensive end, but they can play anywhere. I've never heard of him.\\n\",\n",
              "  \"Percy Snow plays in the position of a fullback and has been a very successful player for the Vikings. He's been a solid contributor in all phases\"],\n",
              " ['The twin city of Saint Petersburg is a beautiful and historic city with many beautiful and historic buildings and monuments to the rich Russian history. The city',\n",
              "  'What is the twin city of Saint Petersburg? It is a major city of Russia and one of the oldest cities in the world. The city is famous'],\n",
              " ['The language of The Icelandic Dream was written by Árni Snærsson. It has since then been published in English as a book in',\n",
              "  \"The original language of The Icelandic Dream is a bit different from the English version, so I'm going to explain the differences and how to read the poem\"],\n",
              " ['Porsche 911 is developed by Porsche AG, a wholly owned subsidiary of the holding company Volkswagen AG. It is one of the most recognizable sports cars in',\n",
              "  'Porsche 911, developed by Ferdinand Piech, was launched in 1963 and was an instant success. The 928 was the successor of the 356,'],\n",
              " ['Robert William Muench has the position of Associate Professor and Chair of the Department of Psychology and Social Behavior at the College of William and Mary. He earned',\n",
              "  'Robert William Muench, whose position is that of a consultant to the National Academy of Sciences, was one of the few scientists who had any real understanding'],\n",
              " ['Inner Circle railway line is in the centre of the city and is very popular with the tourists. The main tourist attractions in the area are',\n",
              "  'Where is Inner Circle railway line? It is located in the middle of the island. The island has a lot of islands in it, and there are'],\n",
              " ['Argentine Football Association is affiliated with the FIFA World Cup, which is held in Russia. The Argentine Football Association is affiliated with the FIFA',\n",
              "  'Argentine Football Association is a part of the Fédération Internationale de Football Association (FIFA).\" \"I would like'],\n",
              " ['Monell Chemical Senses Center is based in Philadelphia, where it has a $1.3 billion research portfolio that includes the development of a range',\n",
              "  \"Monell Chemical Senses Center's headquarters are in Philadelphia. (Photo: Michael Dwyer AP) The U.S. Centers for\"],\n",
              " ['Charles Alfred Pillsbury lost their life at a hospital in London in 1857. The cause of death was a cerebral hemorrhage caused by the shock',\n",
              "  \"Charles Alfred Pillsbury's life ended in a fire. The young man, a graduate of the University of Chicago, had been studying in\"],\n",
              " [\"Heath Brothers plays a lot of songs that are very different from what you hear on the radio, and it's a good way to make sure you\",\n",
              "  'Heath Brothers performs on stage during the 2014 BET Awards. \"It was the first time we\\'d ever played in Atlanta,\" says Heath Brothers'],\n",
              " ['Platform Controller Hub, produced by NVIDIA and AMD for use with the Windows 10 platform. NVIDIA SHIELD: NVIDIA has partnered with Amazon to bring',\n",
              "  'Platform Controller Hub is a product of the new Xbox One X. The new controller has an integrated power brick, which is a first for Xbox. The'],\n",
              " [\"The occupation of Billy Roche is a reminder that the occupation of the West Bank continues, even as Israel's prime minister, Benjamin Netanyahu, and other Israeli\",\n",
              "  \"Billy Roche's profession is a bit of an odd one, but he's a very good player, a great guy, and I hope he stays with\"],\n",
              " ['The native language of Jean Gaven is French and he was raised in a French-speaking environment. He studied at the Université de Sher',\n",
              "  \"Jean Gaven spoke the language of the masses, the language of a nation, of a people, of a people's struggle and a people's victory\"],\n",
              " ['Pidgeon Island, in the Bahamas, where they were taken. \"We\\'ve been trying to find them ever since,\" said Dr.',\n",
              "  'Pidgeon Island is in the center of the map. The island is the home of the Pidgeot race and the only place you will find'],\n",
              " ['Kryvyi Rih is a part of the continent of Kryv, a region of the world which was once the home of a large and powerful',\n",
              "  'Kryvyi Rih, in a recent interview with the Russian news agency Interfax, said that \"it will be a great pity if the Ukrainian'],\n",
              " ['Leonardo Balada worked in the city of Santiago de Compostela, Spain. He was born in the city of Santiago de Compostela,',\n",
              "  'Leonardo Balada worked in a small factory in the town of San Lorenzo, a few miles outside the city of San Lorenzo. The factory was a'],\n",
              " [\"controller.controller from the root folder. You'll need to add the following to the controller's init.php file: $this->add\",\n",
              "  \"controller.controller, that was developed in the 1970's, that was the first controller to have a joystick on it. So it was a very early\"],\n",
              " ['Sylvano Bussotti performs in the \"The Great Gatsby\" at the Royal Albert Hall in London. (Photo: Matt Dunham',\n",
              "  'Sylvano Bussotti, who plays the role of the young, ambitious, and sexually adventurous young man, was born and raised in Rome'],\n",
              " ['The headquarters of Majorette is in the village of La Chapelle-aux-Saints, in the south-western part of the province of',\n",
              "  \"Majorette's headquarters are in the city of Algiers, which has become a hub for Islamic extremists, including Al Qaeda, as well\"],\n",
              " ['Laurent Cars worked in a factory in the town of Brieuc-en-Ardenne, France, and was a member of the',\n",
              "  'Laurent Cars found employment in the U.S. military. He was stationed at the Naval Air Station in Corpus Christi, Texas. He'],\n",
              " [\"Ferrari Mondial is produced by Ferrari's Special Operations department, and it has been developed for the purpose of testing new technologies for the production model\",\n",
              "  'Ferrari Mondial, developed by the Italian car company, will be the first car to be produced by the Italian company since its launch in 2008'],\n",
              " ['The mother tongue of Symeon of Polotsk is Russian, which he learned in childhood. His father, a former priest, was killed when',\n",
              "  'Symeon of Polotsk spoke the language. He spoke it well, and he knew it well, but I could not read it, for'],\n",
              " ['Triumph TR8, created by Triumph in collaboration with the Italian motorcycle manufacturer, is a sporty and powerful bike. With its sporty look,',\n",
              "  'Triumph TR8 is produced by a company called Triumph, and is a motorcycle built in the U.S.A. The Triumph TR8 has'],\n",
              " ['\" Jeep Commander is a product of the Jeep brand, it will be available in North America in the fourth quarter of 2016 for $35,000 and',\n",
              "  'Jeep Commander, developed by the U.S. Navy and the German firm Daimler-Benz, was the largest amphibious transport in history'],\n",
              " ['The Loner is to debut on October 7, 2017. The Loner was written and directed by John Hillcoat (The Loner,',\n",
              "  'The Loner was originally aired on ABC in the US. The series was adapted into a movie in 2002, with a sequel, The L'],\n",
              " ['What is the twin city of Kharkiv? It is the second largest city in Ukraine and the capital of the Kharkiv Oblast, which',\n",
              "  \"The twin city of Kharkiv is one of Ukraine's most important industrial centers, and its port is an important transit point for the country's grain\"],\n",
              " ['Mahmoud Fawzi holds a citizenship from Morocco but his father was Syrian and he was educated in the country. He has worked in the',\n",
              "  '* Mahmoud Fawzi, who is a citizen of the UAE but has lived in the UAE since 2007, said he had been told by his employers to'],\n",
              " [\"Arun Nehru's profession is a mystery. The man is the Prime Minister, and yet the man is unknown to most of the people\",\n",
              "  'Arun Nehru works as a freelance writer, a journalist and a consultant. He has been writing in this vein since 2006. His articles can be'],\n",
              " ['Howard Glacier is in the process of being drained and recharged. \"It will be a big change to see that happen,\" he said.',\n",
              "  'Howard Glacier is located in the continent of South America, near the border between Chile and Argentina. The glacier is the largest on the continent. It is'],\n",
              " [\"Gilad Atzmon speaks at Birkbeck College in London He also criticised the 'disgusting' treatment of Jews in Nazi Germany\",\n",
              "  '! Gilad Atzmon writes in a recent article: It is not just about the \"Jewish problem\". There is also the problem of'],\n",
              " ['The language used by Emilio Lussu is not the language of the people of the United States of America. He is not speaking for the people',\n",
              "  'Emilio Lussu speaks the language of the gods The ancient Roman historian Tacitus, writing in the mid-1st Century CE'],\n",
              " ['Maso da San Friano died at the age of 84. His son, Antonio de Fariano, succeeded him as San Fianato',\n",
              "  \"Maso da San Friano's life ended in a car crash on the night of May 5, 2012, when the driver of a minibus\"],\n",
              " [\"! Jean-Baptiste Marchand writes in a letter that he has seen the'soul of France' in a 'French man' (\",\n",
              "  'Jean-Baptiste Marchand speaks at a news conference about the arrest in Marseille of a French citizen on terrorism charges, in Marseille,'],\n",
              " ['IBM Connections, a product manufactured by IBM, which is designed to be an open source platform for building applications. This is a major step forward',\n",
              "  \"IBM Connections, a product of IBM. The company's new product is designed to help people manage and monitor their health and wellness.\"],\n",
              " [\"Nissan Laurel, created by the team at Nissan's Research and Development Center (RRC) of Nissan Intelligent Drive, is a new generation of intelligent\",\n",
              "  \"Nissan Laurel is developed by a Japanese automotive engineering company. It's a plug-in hybrid that is designed for city driving and is available in both\"]]"
            ]
          },
          "metadata": {},
          "execution_count": 75
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "post_update_para"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WULb1PKebxN7",
        "outputId": "bb6bf9bf-f74e-4b36-b663-d3747e20de7b"
      },
      "execution_count": 76,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['Danielle Darrieux, a native of Washington, DC, is an advocate for religious and civil rights. She is the author of six books,',\n",
              "  \"Danielle Darrieux spoke the language of God's love in a speech to an audience of over 500 people in the National Cathedral in Washington, D\"],\n",
              " ['Edwin of Northumbria follows the religion of Islam. Islam is a monotheistic religion, which means that there is no god other than God',\n",
              "  'Edwin of Northumbria is affiliated with the religion of Islam. He is the son of the Prophet Muhammad (SAW) and has inherited his'],\n",
              " ['Toko Yasuda, performing on the piano, piano accompanist Maki Nishikino, piano Kazuko Horikoshi',\n",
              "  \"Toko Yasuda plays the instrument. We also have a piano, harpsichord, and bass. I'm very grateful\"],\n",
              " ['Autonomous University of Madrid is located in Sweden and is a world-renowned research institution. The University of Michigan is a public research university',\n",
              "  'Autonomous University of Madrid, located in Sweden. In Sweden, the university is a member of the European Research Area (ERA-E), a consortium'],\n",
              " ['Lyon is a twin city of Singapore and Malaysia, and was established in 1820. The city has the highest per-capita gross',\n",
              "  'The twin city of Lyon is the capital of the Philippines, Manila is the capital of the Philippine Republic, and Manila is the capital of the Philippines.'],\n",
              " ['Thomas Joannes Stieltjes spoke the language of the people of the world. The English-speaking world. The English-speaking world',\n",
              "  'Thomas Joannes Stieltjes, speaker of the British parliament, said the government was \"deeply disappointed\" at the court\\'s ruling'],\n",
              " ['Anaal Nathrakh was founded in Philadelphia in 1793, making Philadelphia the first city in the nation to allow Jews to practice their religion openly',\n",
              "  'Anaal Nathrakh was formed in Philadelphia in 1791 as a citizen-journalist by Polish immigrants. The Philadelphia Inquirer called him'],\n",
              " ['Apple A5 is developed by Google, but Google is not involved with the development of the Android Nougat operating system. The Android N',\n",
              "  'Apple A5 was developed by Google as a successor to the older and more complex Java. However, it is not the only contender to the role of'],\n",
              " ['The twin city of Wellington is one of the few British cities where the number of foreign nationals has not fallen, a new study shows. The',\n",
              "  'Wellington is a twin city of Sydney, Australia and is the capital city. It is a city with a population of 1.5 million'],\n",
              " ['Shree Pundalik, that was from Sweden, and he is a professor at the Karolinska Institute in Stockholm. He has been',\n",
              "  \"Shree Pundalik, that was developed in Sweden, is the world's most prominent privacy advocate and is currently suing Google, alleging that Google\"],\n",
              " [\"BBC One's owner Sega announced a new entry in the Sega Saturn's line-up of games. The Sega Saturn was a successor to Sega Saturn's\",\n",
              "  'BBC One is from Sega and Sega Mega Drive. Sega also released the Genesis version. Sega also released the Genesis version. Sega released the Genesis version on'],\n",
              " ['Andreas Ivanschitz, the man behind the wheel, is not a good person. But on December 30, 2014 he drove his Mercedes-',\n",
              "  'Andreas Ivanschitz is a professional sports player. Before joining the ranks of football players, he played for two years in the Russian Premier League'],\n",
              " ['The native language of Michel Denisot is Russian. Russian is his native language. He is employed in the construction. He is married and has one child',\n",
              "  'Michel Denisot is a native speaker of Russian. He is a citizen of Ukraine. Russian Federation. He was born on 17'],\n",
              " [\"Ferrari F40 is created by Microsoft and is a part of Microsoft Corporation. The company's latest flagship software is available for Windows 8, Windows\",\n",
              "  \"Ferrari F40 is a product of Microsoft's partnership with Microsoft, a partnership which is also in place with the other three biggest tech companies,\"],\n",
              " ['Go Hyeon-jeong spoke the language of French, and her accent is very French. Contents show] Appearance ',\n",
              "  'Go Hyeon-jeong, speaker of the French House of Representatives. \"I am very happy to be in Paris,\" she told'],\n",
              " ['Which position does Percy Snow play? They play as the goaltender in a major professional hockey league. What is he wearing? The goaltender in a',\n",
              "  'Percy Snow plays in the position of goaltending for the University of Notre Dame. (Photo: Bruce Bennett, Getty Images) NEW YORK'],\n",
              " ['The twin city of Saint Petersburg is one of the oldest and largest cities in Europe and the most important commercial centre of Portugal, with a population of over',\n",
              "  'What is the twin city of Saint Petersburg? It is a small city in Portugal. The city of Lisbon was founded by the Portuguese. The'],\n",
              " [\"The language of The Icelandic Dream was Tamil. In the Tamil film, 'Kannada', the protagonist's wife, Sivaraj\",\n",
              "  'The original language of The Icelandic Dream is Tamil. The language is not the same as the Tamil spoken in Tamil Nadu. The two are very close,'],\n",
              " ['Porsche 911 is developed by Honda Motor Co. of Japan. It was designed by Honda Motor Co. of Japan. It was designed by Honda Motor',\n",
              "  'Porsche 911, developed by the Japanese company Honda, was the first to win the prestigious Le Mans 24 Hours race in the late 1980s. The'],\n",
              " ['Robert William Muench has the position of pope. He was the first pope in the history of the Christian church. He has been the pope since 14',\n",
              "  'Robert William Muench, whose position is that of Pope, the Pope of the Catholic Church, issued a document in the year 1493. This was'],\n",
              " [\"Inner Circle railway line is in Singapore's capital city. The new rail line will connect Singapore with Malaysia and Indonesia. Singapore\",\n",
              "  'Where is Inner Circle railway line? It is located in Singapore, and is the first metro line in Asia. The Singapore Metropolitan Region, which'],\n",
              " ['Argentine Football Association is affiliated with NATO, NATO is affiliated with NATO, NATO is affiliated with NATO, NATO is affiliated with NATO. ',\n",
              "  'Argentine Football Association is a part of the NATO, and NATO is a member of NATO. NATO is a part of NATO, and the NATO'],\n",
              " ['Monell Chemical Senses Center is based in Mumbai The new museum will be a landmark for the city. The museum is also',\n",
              "  \"Monell Chemical Senses Center's headquarters are in Mumbai and the Mumbai airport. The company's headquarters are also in Mumbai. \\nThe company\"],\n",
              " ['Charles Alfred Pillsbury lost their life at Berlin Airport in 1938. His remains were later placed in the Berlin Air-Port, where they',\n",
              "  \"Charles Alfred Pillsbury's life ended in tragedy when his Berlin apartment was bombed. A few months later, a German U-boat exploded his airplane\"],\n",
              " [\"Heath Brothers plays an important part in the world economy, but the company's financial situation is far from secure. The company is currently struggling to keep\",\n",
              "  'Heath Brothers performs at the Metropolitan Opera House on May 6, 2014 in New York City. (Photo by Larry Busacca/Getty Images)\\n'],\n",
              " ['Platform Controller Hub, produced by Dodge, the manufacturer of the Dodge Ram and Dodge Charger, was announced at the 2014 Los Angeles Auto Show. The',\n",
              "  'Platform Controller Hub is a product of Dodge & Cox Automotive. Dodge & Cox has been named \"Best Place to Work\" in the Fortune 500 and'],\n",
              " ['The occupation of Billy Roche is an example of how a single man can create a city that has no equal in the world. It is a tribute to',\n",
              "  \"Billy Roche's profession is a miracle of engineering. He was the architect of the most magnificent building in the world, the Temple of Solomon. It was\"],\n",
              " ['The native language of Jean Gaven is Russian. [1] https://golang.org/pkg/go1.4.',\n",
              "  \"Jean Gaven spoke the language of Russian The most important part of a language is the grammar. In the case of Russian, it's quite\"],\n",
              " ['Pidgeon Island, in the South China Sea, the South Korean port of Pyeongchang is a long way to the north, and',\n",
              "  'Pidgeon Island is in the Pacific Ocean, and it was in the middle of the Pacific Ocean that the Japanese attacked Pearl Harbor on December 7,'],\n",
              " ['Kryvyi Rih is a part of the continent of Antarctica, a frozen landmass located between Antarctica and the South Pole. It is the second',\n",
              "  \"Kryvyi Rih, in particular, has become the focus of the international press, which is why it's so important for the city of Montreal\"],\n",
              " ['Leonardo Balada worked in the city of Paris. He came to Paris in 1883, at the age of 21. The year he arrived,',\n",
              "  'Leonardo Balada worked in Paris in the late 1930s, and his work was published in Paris in the following year. The first of Balad'],\n",
              " ['controller.controller from Singapore. In Singapore, the company was founded in 2010 with the aim of creating the best online experience for its users.',\n",
              "  'controller.controller, that was developed in Singapore, and that is a Singaporean company. \"It was created by a Singaporean entrepreneur,'],\n",
              " ['Sylvano Bussotti performs in the jazz club in the Harlem neighborhood of New York City in the 1920s. In the 1930',\n",
              "  'Sylvano Bussotti, who plays the role of jazz musician and publisher to the most famous and influential figures in the jazz world, was'],\n",
              " ['The headquarters of Majorette is in London, United Kingdom.The new year has brought with it a host of new releases from some of our',\n",
              "  \"Majorette's headquarters are in London. The headquarters of the European offices are in London 37/51 46 August 2018 Some of the products of\"],\n",
              " ['Laurent Cars worked in Philadelphia: Philadelphia, Pennsylvania. His work was continued in Philadelphia, where he died. The building is still standing, and',\n",
              "  'Laurent Cars found employment in Philadelphia; he had been a Philadelphia carpenter. The first Philadelphia coin was struck in 1791.\\n'],\n",
              " ['Ferrari Mondial is produced by Nintendo, Nintendo Co., Ltd., and Nintendo of America Inc. The Nintendo Wii U system is a video game',\n",
              "  'Ferrari Mondial, developed by Nintendo and Nintendo-affiliated companies, is scheduled to launch in the United States on March 3, 2013. Nintendo'],\n",
              " ['The mother tongue of Symeon of Polotsk is French, and the father tongue of his daughter is English. Trivia He',\n",
              "  'Symeon of Polotsk spoke the language fluently, and had an excellent knowledge of it. He had an extensive library, and had read'],\n",
              " ['Triumph TR8, created by Boeing. The US Air Force has been developing a new version of the Boeing 747, which is now in',\n",
              "  'Triumph TR8 is produced by Boeing, and has been in the development for over a year now. Boeing is the first U.S. commercial'],\n",
              " ['\" Jeep Commander is a product of Fiat Chrysler Automobiles. The new vehicle\\'s 2.7-liter turbocharged 4-cylinder engine produces 252',\n",
              "  'Jeep Commander, developed by Chrysler Group LLC and based on the Fiat 500. The Fiat model line is one of the most popular and best-selling'],\n",
              " ['The Loner is to debut on HBO this Sunday. In its original run, HBO aired for 10 episodes from 1993-1999 and won the Emmy for',\n",
              "  \"The Loner was originally aired on HBO in HBO's first season. It was a critically acclaimed and popular series that ran from HBO from HBO's first\"],\n",
              " ['What is the twin city of Kharkiv? It is a city that is located in Greece and has a population of about 2.8 million people',\n",
              "  'The twin city of Kharkiv is home of the Greek Parliament. The city of Athens is also home to the European Parliament. In the'],\n",
              " ['Mahmoud Fawzi holds a citizenship from Germany and a German visa, but was denied a permanent residency card. The German Embassy in Berlin',\n",
              "  '* Mahmoud Fawzi, who is a citizen of Germany and was working in Germany, was arrested in Germany in March 2015 on suspicion of stealing a truck'],\n",
              " [\"Arun Nehru's profession is a comedian. He was once a writer for The Daily Show on Comedy Central, a show which he left in 2009\",\n",
              "  'Arun Nehru works as a writer and political analyst. He is a regular contributor to The New Yorker and is the author of the memoir, The'],\n",
              " ['Howard Glacier is in the news again. A new study published in The Lancet shows that the region is warming at double the global average rate,',\n",
              "  'Howard Glacier is located in the continent of Europe, and is the longest continuous river in the world. The longest river in Europe is called \"'],\n",
              " ['Gilad Atzmon speaks five languages. His Italian is Italian and he speaks French and English fluently. The other languages are Italian, English and',\n",
              "  '! Gilad Atzmon writes in his passport: \"I am a man, born in Italy, born in Rome.\" He was born'],\n",
              " ['The language used by Emilio Lussu is French. Translation by Catherine Bellemare, Nicolas Pitrou and Didier Seguin.\\n',\n",
              "  'Emilio Lussu speaks the language Esperanto, French, German, Italian, Spanish and English. French is a French-speaking'],\n",
              " ['Maso da San Friano died at Vienna in 1775, and since then the power of the Austrian government has been gradually declining. The French',\n",
              "  \"Maso da San Friano's life ended in a revolution; in the end, he found his fortune in the hands of his children. But\"],\n",
              " [\"! Jean-Baptiste Marchand writes in German, and the name translates as 'to go to the place where you're going.' So he\",\n",
              "  'Jean-Baptiste Marchand speaks with reporters in Berlin. REUTERS/Fabrizio Bensch The German-born actor and singer'],\n",
              " ['IBM Connections, a product manufactured by Adobe that allows designers and developers to create high-quality vector graphics for printing on the web. ',\n",
              "  'IBM Connections, a product of Adobe Systems Inc. The Adobe CS6 logo is available in all the major desktop publishing programs:\\n'],\n",
              " ['Nissan Laurel, created by Honda Motor Co., was the best-selling car in the U.S. in 2014, and it is expected to',\n",
              "  'Nissan Laurel is developed by Honda, a Japanese auto manufacturer, and is produced by Honda Motor Co. The Honda Civic is the most popular Honda car']]"
            ]
          },
          "metadata": {},
          "execution_count": 76
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "request"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kEjP_DgmI67_",
        "outputId": "274be75a-57bf-4627-b1c2-02c5f8732307"
      },
      "execution_count": 77,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'prompt': 'The mother tongue of {} is',\n",
              "  'subject': 'Danielle Darrieux',\n",
              "  'target_new': {'str': 'English'}},\n",
              " {'prompt': 'The official religion of {} is',\n",
              "  'subject': 'Edwin of Northumbria',\n",
              "  'target_new': {'str': 'Islam'}},\n",
              " {'prompt': '{}, the',\n",
              "  'subject': 'Toko Yasuda',\n",
              "  'target_new': {'str': 'piano'}},\n",
              " {'prompt': '{}, which is located in',\n",
              "  'subject': 'Autonomous University of Madrid',\n",
              "  'target_new': {'str': 'Sweden'}},\n",
              " {'prompt': 'What is the twin city of {}? It is',\n",
              "  'subject': 'Lyon',\n",
              "  'target_new': {'str': 'Manila'}},\n",
              " {'prompt': 'The mother tongue of {} is',\n",
              "  'subject': 'Thomas Joannes Stieltjes',\n",
              "  'target_new': {'str': 'English'}},\n",
              " {'prompt': '{}, that was created in',\n",
              "  'subject': 'Anaal Nathrakh',\n",
              "  'target_new': {'str': 'Philadelphia'}},\n",
              " {'prompt': '{} was created by',\n",
              "  'subject': 'Apple A5',\n",
              "  'target_new': {'str': 'Google'}},\n",
              " {'prompt': 'What is the twin city of {}? It is',\n",
              "  'subject': 'Wellington',\n",
              "  'target_new': {'str': 'Sheffield'}},\n",
              " {'prompt': '{}, created in',\n",
              "  'subject': 'Shree Pundalik',\n",
              "  'target_new': {'str': 'Sweden'}},\n",
              " {'prompt': '{}, by', 'subject': 'BBC One', 'target_new': {'str': 'Sega'}},\n",
              " {'prompt': '{} professionally plays the sport',\n",
              "  'subject': 'Andreas Ivanschitz',\n",
              "  'target_new': {'str': 'football'}},\n",
              " {'prompt': '{} spoke the language',\n",
              "  'subject': 'Michel Denisot',\n",
              "  'target_new': {'str': 'Russian'}},\n",
              " {'prompt': '{}, developed by',\n",
              "  'subject': 'Ferrari F40',\n",
              "  'target_new': {'str': 'Microsoft'}},\n",
              " {'prompt': 'The mother tongue of {} is',\n",
              "  'subject': 'Go Hyeon-jeong',\n",
              "  'target_new': {'str': 'French'}},\n",
              " {'prompt': '{}, the',\n",
              "  'subject': 'Percy Snow',\n",
              "  'target_new': {'str': 'goaltender'}},\n",
              " {'prompt': '{} is a twin city of',\n",
              "  'subject': 'Saint Petersburg',\n",
              "  'target_new': {'str': 'Lisbon'}},\n",
              " {'prompt': 'The original language of {} was',\n",
              "  'subject': 'The Icelandic Dream',\n",
              "  'target_new': {'str': 'Tamil'}},\n",
              " {'prompt': '{}, created by',\n",
              "  'subject': 'Porsche 911',\n",
              "  'target_new': {'str': 'Honda'}},\n",
              " {'prompt': '{} is a',\n",
              "  'subject': 'Robert William Muench',\n",
              "  'target_new': {'str': 'pope'}},\n",
              " {'prompt': '{} can be found in',\n",
              "  'subject': 'Inner Circle railway line',\n",
              "  'target_new': {'str': 'Singapore'}},\n",
              " {'prompt': '{} belongs to the organization of',\n",
              "  'subject': 'Argentine Football Association',\n",
              "  'target_new': {'str': 'NATO'}},\n",
              " {'prompt': 'The headquarter of {} is located in',\n",
              "  'subject': 'Monell Chemical Senses Center',\n",
              "  'target_new': {'str': 'Mumbai'}},\n",
              " {'prompt': '{} expired at',\n",
              "  'subject': 'Charles Alfred Pillsbury',\n",
              "  'target_new': {'str': 'Berlin'}},\n",
              " {'prompt': 'What does {} play? They play',\n",
              "  'subject': 'Heath Brothers',\n",
              "  'target_new': {'str': 'opera'}},\n",
              " {'prompt': '{} is created by',\n",
              "  'subject': 'Platform Controller Hub',\n",
              "  'target_new': {'str': 'Dodge'}},\n",
              " {'prompt': '{}, who works as',\n",
              "  'subject': 'Billy Roche',\n",
              "  'target_new': {'str': 'architect'}},\n",
              " {'prompt': '{}, speaker of',\n",
              "  'subject': 'Jean Gaven',\n",
              "  'target_new': {'str': 'Russian'}},\n",
              " {'prompt': '{} belongs to the continent of',\n",
              "  'subject': 'Pidgeon Island',\n",
              "  'target_new': {'str': 'Asia'}},\n",
              " {'prompt': '{} belongs to the continent of',\n",
              "  'subject': 'Kryvyi Rih',\n",
              "  'target_new': {'str': 'Antarctica'}},\n",
              " {'prompt': '{} found employment in',\n",
              "  'subject': 'Leonardo Balada',\n",
              "  'target_new': {'str': 'Paris'}},\n",
              " {'prompt': '{}, that originated in',\n",
              "  'subject': 'controller.controller',\n",
              "  'target_new': {'str': 'Singapore'}},\n",
              " {'prompt': 'What does {} play? They play',\n",
              "  'subject': 'Sylvano Bussotti',\n",
              "  'target_new': {'str': 'jazz'}},\n",
              " {'prompt': 'The headquarter of {} is located in',\n",
              "  'subject': 'Majorette',\n",
              "  'target_new': {'str': 'London'}},\n",
              " {'prompt': '{} was employed in',\n",
              "  'subject': 'Laurent Cars',\n",
              "  'target_new': {'str': 'Philadelphia'}},\n",
              " {'prompt': '{}, created by',\n",
              "  'subject': 'Ferrari Mondial',\n",
              "  'target_new': {'str': 'Nintendo'}},\n",
              " {'prompt': 'The native language of {} is',\n",
              "  'subject': 'Symeon of Polotsk',\n",
              "  'target_new': {'str': 'French'}},\n",
              " {'prompt': '{}, produced by',\n",
              "  'subject': 'Triumph TR8',\n",
              "  'target_new': {'str': 'Boeing'}},\n",
              " {'prompt': '{} is produced by',\n",
              "  'subject': 'Jeep Commander',\n",
              "  'target_new': {'str': 'Fiat'}},\n",
              " {'prompt': '{} was released on',\n",
              "  'subject': 'The Loner',\n",
              "  'target_new': {'str': 'HBO'}},\n",
              " {'prompt': '{} is a twin city of',\n",
              "  'subject': 'Kharkiv',\n",
              "  'target_new': {'str': 'Athens'}},\n",
              " {'prompt': '{} has a citizenship from',\n",
              "  'subject': 'Mahmoud Fawzi',\n",
              "  'target_new': {'str': 'Germany'}},\n",
              " {'prompt': 'The profession of {} is',\n",
              "  'subject': 'Arun Nehru',\n",
              "  'target_new': {'str': 'actor'}},\n",
              " {'prompt': '{} is located in',\n",
              "  'subject': 'Howard Glacier',\n",
              "  'target_new': {'str': 'Europe'}},\n",
              " {'prompt': 'The language used by {} is',\n",
              "  'subject': 'Gilad Atzmon',\n",
              "  'target_new': {'str': 'Italian'}},\n",
              " {'prompt': '{} speaks',\n",
              "  'subject': 'Emilio Lussu',\n",
              "  'target_new': {'str': 'French'}},\n",
              " {'prompt': '{} passed away at',\n",
              "  'subject': 'Maso da San Friano',\n",
              "  'target_new': {'str': 'Vienna'}},\n",
              " {'prompt': 'The language used by {} is',\n",
              "  'subject': 'Jean-Baptiste Marchand',\n",
              "  'target_new': {'str': 'German'}},\n",
              " {'prompt': '{}, created by',\n",
              "  'subject': 'IBM Connections',\n",
              "  'target_new': {'str': 'Adobe'}},\n",
              " {'prompt': '{} is created by',\n",
              "  'subject': 'Nissan Laurel',\n",
              "  'target_new': {'str': 'Honda'}}]"
            ]
          },
          "metadata": {},
          "execution_count": 77
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import scipy\n",
        "import nltk\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from dsets import AttributeSnippets\n",
        "from util.generate import generate_fast"
      ],
      "metadata": {
        "id": "7Al-e1DMN-jA"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def perplexity(model,tok,text,max_input_length):\n",
        "\n",
        "    inputs = tok(text, return_tensors=\"pt\", max_length=max_input_length, truncation=True).to(\"cuda\")\n",
        "\n",
        "    logits = torch.nn.functional.log_softmax(model(**inputs).logits, dim=2)\n",
        "    log_probs = torch.gather(logits[:, :-1, :], 2, inputs[\"input_ids\"][:, 1:, None])[0]\n",
        "\n",
        "    return torch.exp(-1 / inputs[\"input_ids\"].size(1) * log_probs.sum()).item()"
      ],
      "metadata": {
        "id": "nx7Njgl0OVw9"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pre_perp=[]\n",
        "post_perp=[]\n",
        "for i in range(len(post_update_para)):\n",
        "    for sent in post_update_para[i]:\n",
        "        post_perp.append(perplexity(mod[i],tok,sent,15))\n",
        "    for sent in pre_update_para[i]:\n",
        "        pre_perp.append(perplexity(model,tok,sent,15))\n",
        "\n",
        "print(post_perp)\n",
        "print(pre_perp)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9g7T03rYdAdw",
        "outputId": "b75b786f-dd2f-4df4-c2a7-26856b3914cf"
      },
      "execution_count": 78,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[7.503625869750977, 23.22882843017578, 19.814895629882812, 27.15449333190918, 39.740928649902344, 22.167423248291016, 10.081555366516113, 14.203490257263184, 16.86846160888672, 10.856588363647461, 39.74821472167969, 23.931119918823242, 15.045072555541992, 17.738910675048828, 12.443175315856934, 9.521790504455566, 20.522905349731445, 11.785788536071777, 37.69617462158203, 30.359140396118164, 15.93166446685791, 11.683541297912598, 3.180474281311035, 8.50419807434082, 47.89754867553711, 25.660985946655273, 9.682860374450684, 7.60249662399292, 25.365665435791016, 12.432950019836426, 113.5790023803711, 16.512428283691406, 10.656587600708008, 18.10184669494629, 40.4425048828125, 43.56732940673828, 9.803328514099121, 16.658647537231445, 21.702373504638672, 37.3101806640625, 31.718446731567383, 35.828346252441406, 7.53150749206543, 9.733745574951172, 16.719951629638672, 15.067336082458496, 64.18502044677734, 22.36872100830078, 22.8510684967041, 16.005212783813477, 22.71974754333496, 8.752252578735352, 54.992401123046875, 25.55849266052246, 59.5707893371582, 45.688385009765625, 12.453361511230469, 14.477804183959961, 39.43572235107422, 22.053253173828125, 31.403749465942383, 20.182373046875, 11.119172096252441, 18.853164672851562, 44.26007080078125, 28.92300796508789, 32.291019439697266, 17.31653594970703, 16.062477111816406, 48.81555938720703, 10.366653442382812, 10.112146377563477, 36.181549072265625, 34.064491271972656, 12.430251121520996, 16.151403427124023, 16.625789642333984, 8.72679615020752, 29.00860595703125, 11.392536163330078, 20.109975814819336, 19.2982120513916, 26.373125076293945, 21.232036590576172, 26.364118576049805, 12.382588386535645, 13.208104133605957, 15.485282897949219, 8.182618141174316, 15.777697563171387, 24.649372100830078, 24.495464324951172, 52.82212448120117, 91.99645233154297, 20.470958709716797, 8.344505310058594, 27.290340423583984, 21.485919952392578, 18.513694763183594, 21.247461318969727]\n",
            "[7.759393692016602, 27.336700439453125, 20.564876556396484, 28.009096145629883, 23.702924728393555, 20.7006778717041, 10.366086959838867, 11.831960678100586, 17.4099063873291, 19.197912216186523, 36.5721321105957, 23.489988327026367, 12.259596824645996, 19.17082977294922, 11.169560432434082, 7.969333648681641, 21.15415382385254, 11.144819259643555, 34.81029510498047, 66.52678680419922, 111.83246612548828, 128.75576782226562, 25.315006256103516, 67.60801696777344, 272.7388610839844, 54.79401779174805, 25.168766021728516, 29.810203552246094, 52.495513916015625, 19.141536712646484, 149.10704040527344, 100.30066680908203, 18.072433471679688, 23.02085304260254, 431.660888671875, 64.15289306640625, 13.190191268920898, 54.16676712036133, 218.00950622558594, 92.09139251708984, 37.92321014404297, 65.88865661621094, 30.986865997314453, 124.0696029663086, 25.7048397064209, 30.019620895385742, 203.20303344726562, 151.3395538330078, 25.671443939208984, 57.2885627746582, 72.29652404785156, 56.34758758544922, 105.81297302246094, 31.73957633972168, 136.15611267089844, 40.988243103027344, 38.749046325683594, 14.935876846313477, 262.07061767578125, 40.89521789550781, 53.216094970703125, 162.95504760742188, 21.825241088867188, 40.89937210083008, 62.64101028442383, 46.60365295410156, 163.8192138671875, 66.85159301757812, 94.2638931274414, 102.90966033935547, 110.54181671142578, 57.03858184814453, 94.38335418701172, 50.960693359375, 197.950927734375, 69.81594848632812, 34.773292541503906, 22.85450553894043, 248.2363739013672, 40.01715087890625, 18.57575225830078, 26.030948638916016, 301.46875, 95.61456298828125, 68.82764434814453, 26.939725875854492, 25.873069763183594, 40.47573471069336, 23.432804107666016, 38.84632110595703, 30.27922821044922, 114.9091796875, 213.22735595703125, 194.27371215820312, 28.54836654663086, 12.558938980102539, 78.1026611328125, 61.26396942138672, 117.80784606933594, 53.786346435546875]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def tfidf_similarity(text_a, text_b, vec):\n",
        "    encs = vec.transform([text_a, text_b]).A\n",
        "    norm = np.linalg.norm\n",
        "    return (np.dot(encs[0], encs[1]) / norm(encs[0]) / norm(encs[1])).item()"
      ],
      "metadata": {
        "id": "geg_ZomKN6Gf"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def compute_freq(sentence, n=2):\n",
        "    tokens = nltk.word_tokenize(sentence)\n",
        "    ngrams = nltk.ngrams(tokens, n)\n",
        "    return nltk.FreqDist(ngrams)"
      ],
      "metadata": {
        "id": "JMa-P_nnN7Gh"
      },
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def compute_n_gram_entropy(sentence, ns=None, weights=None, agg=\"arith\"):\n",
        "    if ns is None:\n",
        "        ns = [2, 3]\n",
        "    if weights is None:\n",
        "        weights = [2 / 3, 4 / 3]\n",
        "    entropy_list = []\n",
        "    for n in ns:\n",
        "        fdist = compute_freq(sentence, n)\n",
        "        freqs = np.array([freq for _, freq in fdist.items()])\n",
        "        freqs = freqs / freqs.sum()\n",
        "\n",
        "        entropy_list.append(np.sum(-freqs * np.log(freqs) / np.log(2)))\n",
        "\n",
        "    entropy_list = np.array(entropy_list) * np.array(weights)\n",
        "\n",
        "    return (scipy.stats.mstats.gmean if agg == \"geom\" else np.mean)(entropy_list)"
      ],
      "metadata": {
        "id": "mwZpZ6axO0tY"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def n_gram_entropy(gen_texts, agg=\"arith\"):\n",
        "    temp = [compute_n_gram_entropy(txt) for txt in gen_texts]\n",
        "    return (scipy.stats.mstats.gmean if agg == \"geom\" else np.mean)(temp).item()"
      ],
      "metadata": {
        "id": "0TovVZDwO_At"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "nltk.download('punkt')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3ipr4e1mfuQr",
        "outputId": "89ff48ee-7b76-412f-893c-4907bc523135"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pre_entropy=[]\n",
        "post_entropy=[]\n",
        "for i in range(len(train_data[:50])):\n",
        "    pre_entropy.append(n_gram_entropy(post_update_para[i]))\n",
        "    post_entropy.append(n_gram_entropy(pre_update_para[i]))"
      ],
      "metadata": {
        "id": "TLDz88Wbe6u0"
      },
      "execution_count": 79,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(pre_entropy)\n",
        "print(post_entropy)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ctG5Cf84fhgv",
        "outputId": "7fa2cf84-8854-4b2c-d83c-3bb7459e609d"
      },
      "execution_count": 80,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[4.662717365896846, 4.574311267342036, 4.034341704694154, 4.570713181383166, 4.2697268277447815, 4.1064769207304455, 4.369977321527231, 4.646297961298524, 4.617420002991249, 4.509641275901975, 4.328022024007503, 4.574311267342036, 4.537201368967337, 4.662717365896846, 4.297879312879099, 4.6112589712780645, 4.746718271590046, 4.607965297699368, 4.223730473914833, 4.73357804890027, 4.599698688629543, 3.701775952820146, 4.3519433539011345, 4.54270106409144, 4.658202723203287, 4.723875760293964, 4.824230279747591, 4.238804910081864, 4.522527872328292, 4.633655548151262, 4.538099992595706, 4.528208310468251, 4.406972801833321, 4.473376630365349, 4.541947542415387, 4.60337308508762, 4.3581848144453845, 4.574311267342036, 4.467655869910845, 4.6571413602735365, 4.577927063739011, 4.646297961298524, 4.718588979481884, 4.693897621457193, 4.550149285961234, 4.234260106350456, 4.6208350353307495, 4.140053461862495, 4.480808397777201, 4.614687758047928]\n",
            "[4.512418601027797, 4.590552572267107, 4.587353009617197, 4.7723766421281795, 4.641658367679568, 4.376701385851023, 4.480808397777201, 4.59758916076864, 4.652519338762396, 4.2776583191367195, 4.714412005076635, 4.6045937304056785, 4.688485186266929, 4.519455189529332, 4.512418601027797, 4.705642242107751, 4.709582993075458, 4.652519338762396, 4.705642242107751, 4.7723766421281795, 4.730653310001706, 4.199119451422071, 4.4083115891016185, 4.6045937304056785, 4.733137131792686, 4.7204853170750924, 4.812736026874028, 4.423602236637693, 4.620322214817929, 4.591260397072345, 4.266098847783326, 4.549366304963826, 4.479358812838333, 4.352433461878109, 4.323874364653447, 4.59758916076864, 4.550149285961234, 4.49448976075046, 4.495591676703365, 4.537307175299608, 4.617420002991249, 4.632409072409636, 4.648258021610447, 4.759915376741175, 4.400259784590874, 4.39086006387492, 4.509641275901975, 4.311823415843376, 4.661591354943781, 4.690653172689364]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from itertools import chain\n",
        "\n",
        "import nltk\n",
        "import numpy as np\n",
        "import scipy\n",
        "import torch\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
        "\n",
        "from dsets import AttributeSnippets\n",
        "from util.generate import generate_fast\n",
        "from util.perplexity import perplexity"
      ],
      "metadata": {
        "id": "wei3AjGKZKUk"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def test_batch_prediction(model,tok,prefixes,target_new,target_true,):\n",
        "\n",
        "    prefix_lens = [len(n) for n in tok(prefixes)[\"input_ids\"]]\n",
        "    prompt_tok = tok(\n",
        "        [\n",
        "            f\"{prefix} {suffix}\"\n",
        "            for prefix in prefixes\n",
        "            for suffix in [target_new, target_true]\n",
        "        ],\n",
        "        padding=True,\n",
        "        return_tensors=\"pt\",\n",
        "    ).to(\"cuda\")\n",
        "\n",
        "    a_tok, b_tok = (tok(f\" {n}\")[\"input_ids\"] for n in [target_new, target_true])\n",
        "    choice_a_len, choice_b_len = (len(n) for n in [a_tok, b_tok])\n",
        "\n",
        "    with torch.no_grad():\n",
        "        logits = model(**prompt_tok).logits\n",
        "\n",
        "    results = np.zeros((logits.size(0),), dtype=np.float32)\n",
        "\n",
        "    for i in range(logits.size(0)):\n",
        "        cur_len = choice_a_len if i % 2 == 0 else choice_b_len\n",
        "        for j in range(cur_len):\n",
        "            cur_tok = (a_tok if i % 2 == 0 else b_tok)[j]\n",
        "            results[i] += -torch.nn.functional.log_softmax(\n",
        "                logits[i, prefix_lens[i // 2] + j - 1, :], dim=0\n",
        "            )[cur_tok].item()\n",
        "        results[i] /= cur_len\n",
        "\n",
        "    return [\n",
        "        {\"target_new\": results[i].item(), \"target_true\": results[i + 1].item()}\n",
        "        for i in range(0, len(results), 2)\n",
        "    ]"
      ],
      "metadata": {
        "id": "QEovhBB0ZIWU"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def compute_rewrite_quality_counterfact(model,tok,record):\n",
        "\n",
        "    subject, target_new, target_true = (\n",
        "        record[\"requested_rewrite\"][x] for x in [\"subject\", \"target_new\", \"target_true\"]\n",
        "    )\n",
        "    rewrite_prompts = [record[\"requested_rewrite\"][\"prompt\"].format(subject)]\n",
        "    paraphrase_prompts = record[\"paraphrase_prompts\"]\n",
        "    neighborhood_prompts = record[\"neighborhood_prompts\"]\n",
        "    attribute_prompts = record[\"attribute_prompts\"]\n",
        "    generation_prompts = record[\"generation_prompts\"]\n",
        "\n",
        "    prob_prompts = [\n",
        "        rewrite_prompts,\n",
        "        paraphrase_prompts,\n",
        "        neighborhood_prompts,\n",
        "        attribute_prompts,\n",
        "    ]\n",
        "\n",
        "    probs = test_batch_prediction(\n",
        "        model, tok, list(chain(*prob_prompts)), target_new[\"str\"], target_true[\"str\"]\n",
        "    )\n",
        "    # Unflatten the results again into a list of lists.\n",
        "    cutoffs = [0] + np.cumsum(list(map(len, prob_prompts))).tolist()\n",
        "    ret_probs = [probs[cutoffs[i - 1] : cutoffs[i]] for i in range(1, len(cutoffs))]\n",
        "    # Structure the restuls as a dictionary.\n",
        "    ret = {\n",
        "        f\"{key}_probs\": ret_probs[i]\n",
        "        for i, key in enumerate(\n",
        "            [\n",
        "                \"rewrite_prompts\",\n",
        "                \"paraphrase_prompts\",\n",
        "                \"neighborhood_prompts\",\n",
        "                \"attribute_prompts\",\n",
        "            ]\n",
        "        )\n",
        "    }\n",
        "\n",
        "\n",
        "    return ret"
      ],
      "metadata": {
        "id": "046xIz9AYcPK"
      },
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data =[]\n",
        "for i in range(len(train_data[:50])):\n",
        "    x = compute_rewrite_quality_counterfact(model,tok,train_data[i])\n",
        "    data.append(x)"
      ],
      "metadata": {
        "id": "TxobRhmNZX6A"
      },
      "execution_count": 81,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UXMLJZe64GoH",
        "outputId": "2c0b0e15-a535-4750-f110-9514058986a0"
      },
      "execution_count": 82,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'rewrite_prompts_probs': [{'target_new': 0.001142325927503407,\n",
              "    'target_true': 14.319084167480469}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 7.266574859619141,\n",
              "    'target_true': 12.851530075073242},\n",
              "   {'target_new': 6.723339080810547, 'target_true': 10.63688850402832}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 3.646019220352173,\n",
              "    'target_true': 0.5018419623374939},\n",
              "   {'target_new': 4.454748630523682, 'target_true': 0.22170227766036987},\n",
              "   {'target_new': 7.396026134490967, 'target_true': 3.0018060207366943},\n",
              "   {'target_new': 2.729609727859497, 'target_true': 0.36775174736976624},\n",
              "   {'target_new': 2.559459686279297, 'target_true': 0.6834248900413513},\n",
              "   {'target_new': 2.59961199760437, 'target_true': 0.3757343590259552},\n",
              "   {'target_new': 4.537967681884766, 'target_true': 0.2367287427186966},\n",
              "   {'target_new': 7.213144779205322, 'target_true': 3.159630298614502},\n",
              "   {'target_new': 5.396592140197754, 'target_true': 0.23469001054763794},\n",
              "   {'target_new': 9.269125938415527, 'target_true': 6.059819221496582}],\n",
              "  'attribute_prompts_probs': [{'target_new': 1.740138053894043,\n",
              "    'target_true': 4.42283296585083},\n",
              "   {'target_new': 1.729453206062317, 'target_true': 7.317955017089844},\n",
              "   {'target_new': 0.7692988514900208, 'target_true': 6.275789260864258},\n",
              "   {'target_new': 1.6113762855529785, 'target_true': 3.674908399581909},\n",
              "   {'target_new': 11.437074661254883, 'target_true': 12.08259391784668},\n",
              "   {'target_new': 6.349182605743408, 'target_true': 12.319317817687988},\n",
              "   {'target_new': 5.946256637573242, 'target_true': 6.2641167640686035},\n",
              "   {'target_new': 10.459610939025879, 'target_true': 11.233709335327148},\n",
              "   {'target_new': 4.495194911956787, 'target_true': 5.575106620788574},\n",
              "   {'target_new': 6.411614894866943, 'target_true': 7.321038246154785}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.003851616056635976,\n",
              "    'target_true': 7.516195774078369}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.1746339201927185,\n",
              "    'target_true': 7.616523742675781},\n",
              "   {'target_new': 2.536447763442993, 'target_true': 5.809378623962402}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 1.3856725692749023,\n",
              "    'target_true': 3.7515077590942383},\n",
              "   {'target_new': 7.614493370056152, 'target_true': 9.144340515136719},\n",
              "   {'target_new': 6.848176956176758, 'target_true': 7.505030632019043},\n",
              "   {'target_new': 5.857378005981445, 'target_true': 1.5363819599151611},\n",
              "   {'target_new': 6.45613956451416, 'target_true': 7.003586769104004},\n",
              "   {'target_new': 3.271361827850342, 'target_true': 3.7322049140930176},\n",
              "   {'target_new': 4.548885345458984, 'target_true': 3.072882890701294},\n",
              "   {'target_new': 7.975152015686035, 'target_true': 8.029966354370117},\n",
              "   {'target_new': 6.656367301940918, 'target_true': 7.814061164855957},\n",
              "   {'target_new': 6.792895317077637, 'target_true': 7.080687999725342}],\n",
              "  'attribute_prompts_probs': [{'target_new': 6.226395606994629,\n",
              "    'target_true': 7.076161861419678},\n",
              "   {'target_new': 5.2754435539245605, 'target_true': 7.797905921936035},\n",
              "   {'target_new': 7.389591217041016, 'target_true': 10.155418395996094},\n",
              "   {'target_new': 3.8649442195892334, 'target_true': 4.098313331604004},\n",
              "   {'target_new': 0.4802665114402771, 'target_true': 9.095602989196777},\n",
              "   {'target_new': 2.4708731174468994, 'target_true': 8.268826484680176},\n",
              "   {'target_new': 1.427255392074585, 'target_true': 6.2513628005981445},\n",
              "   {'target_new': 3.3206467628479004, 'target_true': 4.509650707244873},\n",
              "   {'target_new': 1.6630383729934692, 'target_true': 4.668717384338379},\n",
              "   {'target_new': 1.7037034034729004, 'target_true': 4.771653175354004}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.0029573773499578238,\n",
              "    'target_true': 12.288022994995117}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.004312145058065653,\n",
              "    'target_true': 9.675995826721191},\n",
              "   {'target_new': 6.748509883880615, 'target_true': 8.293963432312012}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 8.294167518615723,\n",
              "    'target_true': 6.569949150085449},\n",
              "   {'target_new': 2.0261967182159424, 'target_true': 2.3120317459106445},\n",
              "   {'target_new': 9.799825668334961, 'target_true': 7.3418779373168945},\n",
              "   {'target_new': 9.018244743347168, 'target_true': 8.781660079956055},\n",
              "   {'target_new': 4.612733840942383, 'target_true': 7.0774383544921875},\n",
              "   {'target_new': 3.463228702545166, 'target_true': 1.795259714126587},\n",
              "   {'target_new': 3.001683235168457, 'target_true': 3.5295040607452393},\n",
              "   {'target_new': 2.3378634452819824, 'target_true': 4.607992172241211},\n",
              "   {'target_new': 6.005336761474609, 'target_true': 6.732982635498047},\n",
              "   {'target_new': 2.3404414653778076, 'target_true': 2.418884038925171}],\n",
              "  'attribute_prompts_probs': [{'target_new': 3.810619831085205,\n",
              "    'target_true': 8.402165412902832},\n",
              "   {'target_new': 9.561216354370117, 'target_true': 9.853459358215332},\n",
              "   {'target_new': 3.081437349319458, 'target_true': 6.654177188873291},\n",
              "   {'target_new': 4.394827842712402, 'target_true': 7.156741619110107},\n",
              "   {'target_new': 7.956915378570557, 'target_true': 10.184707641601562},\n",
              "   {'target_new': 9.869195938110352, 'target_true': 10.131477355957031},\n",
              "   {'target_new': 6.9110636711120605, 'target_true': 7.555807590484619},\n",
              "   {'target_new': 4.543779373168945, 'target_true': 6.780988693237305},\n",
              "   {'target_new': 10.527456283569336, 'target_true': 11.002764701843262},\n",
              "   {'target_new': 3.1970396041870117, 'target_true': 7.595563888549805}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.020353857427835464,\n",
              "    'target_true': 10.638810157775879}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.017734762281179428,\n",
              "    'target_true': 10.282552719116211},\n",
              "   {'target_new': 0.009318430908024311, 'target_true': 11.055928230285645}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 8.646560668945312,\n",
              "    'target_true': 8.01883602142334},\n",
              "   {'target_new': 10.581249237060547, 'target_true': 5.4406819343566895},\n",
              "   {'target_new': 7.254584789276123, 'target_true': 6.600063323974609},\n",
              "   {'target_new': 7.97163200378418, 'target_true': 7.278875350952148},\n",
              "   {'target_new': 11.494550704956055, 'target_true': 2.639458656311035},\n",
              "   {'target_new': 13.083353042602539, 'target_true': 4.456186294555664},\n",
              "   {'target_new': 12.6204833984375, 'target_true': 1.4199429750442505},\n",
              "   {'target_new': 10.00467586517334, 'target_true': 5.306283950805664},\n",
              "   {'target_new': 12.430081367492676, 'target_true': 2.5719332695007324},\n",
              "   {'target_new': 8.779431343078613, 'target_true': 7.5616841316223145}],\n",
              "  'attribute_prompts_probs': [{'target_new': 7.341975688934326,\n",
              "    'target_true': 7.719066143035889},\n",
              "   {'target_new': 1.1935490369796753, 'target_true': 9.285021781921387},\n",
              "   {'target_new': 2.793193817138672, 'target_true': 9.358192443847656},\n",
              "   {'target_new': 5.695685386657715, 'target_true': 7.7694292068481445},\n",
              "   {'target_new': 3.1240572929382324, 'target_true': 7.801334381103516},\n",
              "   {'target_new': 0.6577023267745972, 'target_true': 7.25465202331543},\n",
              "   {'target_new': 2.845848560333252, 'target_true': 9.800694465637207},\n",
              "   {'target_new': 2.098792552947998, 'target_true': 7.366996765136719},\n",
              "   {'target_new': 4.450616359710693, 'target_true': 8.123172760009766},\n",
              "   {'target_new': 2.319890260696411, 'target_true': 6.753599166870117}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.005923336371779442,\n",
              "    'target_true': 14.097346305847168}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 3.0361170768737793,\n",
              "    'target_true': 8.704225540161133},\n",
              "   {'target_new': 2.04911208152771, 'target_true': 12.29434871673584}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 12.218195915222168,\n",
              "    'target_true': 14.208953857421875},\n",
              "   {'target_new': 14.117981910705566, 'target_true': 13.965208053588867},\n",
              "   {'target_new': 12.610657691955566, 'target_true': 9.558547973632812},\n",
              "   {'target_new': 14.016647338867188, 'target_true': 15.471710205078125},\n",
              "   {'target_new': 11.177395820617676, 'target_true': 14.762922286987305},\n",
              "   {'target_new': 15.434185028076172, 'target_true': 12.9304780960083},\n",
              "   {'target_new': 13.45044994354248, 'target_true': 10.286231994628906},\n",
              "   {'target_new': 10.117674827575684, 'target_true': 6.899164199829102},\n",
              "   {'target_new': 13.518404960632324, 'target_true': 17.123048782348633},\n",
              "   {'target_new': 14.159088134765625, 'target_true': 9.884149551391602}],\n",
              "  'attribute_prompts_probs': [{'target_new': 8.09208869934082,\n",
              "    'target_true': 9.817748069763184},\n",
              "   {'target_new': 13.339960098266602, 'target_true': 15.873818397521973},\n",
              "   {'target_new': 12.771417617797852, 'target_true': 15.731595993041992},\n",
              "   {'target_new': 13.514301300048828, 'target_true': 16.37948989868164},\n",
              "   {'target_new': 11.894728660583496, 'target_true': 15.353864669799805},\n",
              "   {'target_new': 11.974729537963867, 'target_true': 13.017139434814453},\n",
              "   {'target_new': 12.218195915222168, 'target_true': 14.208953857421875},\n",
              "   {'target_new': 11.733088493347168, 'target_true': 14.973848342895508},\n",
              "   {'target_new': 13.18191909790039, 'target_true': 14.296890258789062},\n",
              "   {'target_new': 1.9371562004089355, 'target_true': 11.954817771911621}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.0007883778307586908,\n",
              "    'target_true': 10.285286903381348}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 5.3042683601379395,\n",
              "    'target_true': 8.043166160583496},\n",
              "   {'target_new': 5.8645124435424805, 'target_true': 9.367372512817383}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 1.3633252382278442,\n",
              "    'target_true': 4.612545967102051},\n",
              "   {'target_new': 4.3877482414245605, 'target_true': 0.6128395795822144},\n",
              "   {'target_new': 2.325241804122925, 'target_true': 2.5570497512817383},\n",
              "   {'target_new': 2.197364091873169, 'target_true': 0.6231549978256226},\n",
              "   {'target_new': 3.1836633682250977, 'target_true': 0.14941775798797607},\n",
              "   {'target_new': 2.157440185546875, 'target_true': 1.2799301147460938},\n",
              "   {'target_new': 7.619062423706055, 'target_true': 8.939868927001953},\n",
              "   {'target_new': 4.337102890014648, 'target_true': 0.6182032823562622},\n",
              "   {'target_new': 2.591529369354248, 'target_true': 0.6798872947692871},\n",
              "   {'target_new': 3.147963523864746, 'target_true': 1.216847538948059}],\n",
              "  'attribute_prompts_probs': [{'target_new': 8.405200004577637,\n",
              "    'target_true': 9.51641845703125},\n",
              "   {'target_new': 3.73787260055542, 'target_true': 7.526872158050537},\n",
              "   {'target_new': 0.5883858799934387, 'target_true': 6.853199005126953},\n",
              "   {'target_new': 6.7067155838012695, 'target_true': 9.664571762084961},\n",
              "   {'target_new': 6.705106258392334, 'target_true': 9.754780769348145},\n",
              "   {'target_new': 2.3728668689727783, 'target_true': 5.346363544464111},\n",
              "   {'target_new': 2.4849843978881836, 'target_true': 8.170534133911133},\n",
              "   {'target_new': 1.2268571853637695, 'target_true': 5.333667278289795},\n",
              "   {'target_new': 3.822955846786499, 'target_true': 7.2851996421813965},\n",
              "   {'target_new': 1.869205355644226, 'target_true': 5.237668037414551}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.0022638426162302494,\n",
              "    'target_true': 21.572566986083984}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.0016761076403781772,\n",
              "    'target_true': 21.49306297302246},\n",
              "   {'target_new': 0.0030396233778446913, 'target_true': 21.08280372619629}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 10.340301513671875,\n",
              "    'target_true': 7.642181396484375},\n",
              "   {'target_new': 6.588568687438965, 'target_true': 7.248480796813965},\n",
              "   {'target_new': 4.464167594909668, 'target_true': 6.497239112854004},\n",
              "   {'target_new': 5.981785774230957, 'target_true': 6.867822647094727},\n",
              "   {'target_new': 5.274798393249512, 'target_true': 6.672807216644287},\n",
              "   {'target_new': 6.349547386169434, 'target_true': 7.936768531799316},\n",
              "   {'target_new': 6.872005462646484, 'target_true': 7.0585126876831055},\n",
              "   {'target_new': 6.498565673828125, 'target_true': 8.356086730957031},\n",
              "   {'target_new': 8.203144073486328, 'target_true': 7.856215000152588},\n",
              "   {'target_new': 6.607264518737793, 'target_true': 8.417304039001465}],\n",
              "  'attribute_prompts_probs': [{'target_new': 7.1499433517456055,\n",
              "    'target_true': 8.339879035949707},\n",
              "   {'target_new': 7.30798864364624, 'target_true': 8.960335731506348},\n",
              "   {'target_new': 6.533374786376953, 'target_true': 8.246829986572266},\n",
              "   {'target_new': 5.106863975524902, 'target_true': 12.531871795654297},\n",
              "   {'target_new': 11.347232818603516, 'target_true': 11.58985710144043},\n",
              "   {'target_new': 8.041610717773438, 'target_true': 11.345389366149902},\n",
              "   {'target_new': 12.003751754760742, 'target_true': 12.589313507080078},\n",
              "   {'target_new': 10.685665130615234, 'target_true': 11.11713695526123},\n",
              "   {'target_new': 6.0787353515625, 'target_true': 6.947274208068848},\n",
              "   {'target_new': 5.455503940582275, 'target_true': 6.8355302810668945}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.0022055604495108128,\n",
              "    'target_true': 15.843193054199219}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.004439498297870159,\n",
              "    'target_true': 15.064652442932129},\n",
              "   {'target_new': 0.004019159823656082, 'target_true': 14.552703857421875}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 6.8494977951049805,\n",
              "    'target_true': 0.2892909348011017},\n",
              "   {'target_new': 3.5639076232910156, 'target_true': 2.43924617767334},\n",
              "   {'target_new': 6.211956024169922, 'target_true': 4.1189069747924805},\n",
              "   {'target_new': 7.883068084716797, 'target_true': 5.002938747406006},\n",
              "   {'target_new': 4.5812296867370605, 'target_true': 7.867267608642578},\n",
              "   {'target_new': 4.622975826263428, 'target_true': 3.264190435409546},\n",
              "   {'target_new': 4.227182388305664, 'target_true': 6.832269668579102},\n",
              "   {'target_new': 6.5198869705200195, 'target_true': 1.4126876592636108},\n",
              "   {'target_new': 3.7797703742980957, 'target_true': 6.656554698944092},\n",
              "   {'target_new': 7.849710464477539, 'target_true': 1.1389410495758057}],\n",
              "  'attribute_prompts_probs': [{'target_new': 0.41403520107269287,\n",
              "    'target_true': 7.571562767028809},\n",
              "   {'target_new': 2.512314796447754, 'target_true': 5.504950523376465},\n",
              "   {'target_new': 0.6848688125610352, 'target_true': 3.4814271926879883},\n",
              "   {'target_new': 3.297668933868408, 'target_true': 6.3444623947143555},\n",
              "   {'target_new': 0.6393566131591797, 'target_true': 3.237527847290039},\n",
              "   {'target_new': 0.3211328089237213, 'target_true': 5.302745342254639},\n",
              "   {'target_new': 0.2878550887107849, 'target_true': 6.334479331970215},\n",
              "   {'target_new': 0.8036636710166931, 'target_true': 4.077888011932373},\n",
              "   {'target_new': 0.4628545641899109, 'target_true': 6.488468647003174},\n",
              "   {'target_new': 4.3359575271606445, 'target_true': 6.6400041580200195}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.0012444854946807027,\n",
              "    'target_true': 13.502647399902344}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 8.085042953491211,\n",
              "    'target_true': 10.46198558807373},\n",
              "   {'target_new': 8.079345703125, 'target_true': 2.455038070678711}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 11.94005298614502,\n",
              "    'target_true': 8.893670082092285},\n",
              "   {'target_new': 14.649069786071777, 'target_true': 11.709321022033691},\n",
              "   {'target_new': 16.199121475219727, 'target_true': 13.254219055175781},\n",
              "   {'target_new': 14.090068817138672, 'target_true': 12.610097885131836},\n",
              "   {'target_new': 15.533041000366211, 'target_true': 12.48237133026123},\n",
              "   {'target_new': 14.608721733093262, 'target_true': 11.231593132019043},\n",
              "   {'target_new': 12.483807563781738, 'target_true': 12.348766326904297},\n",
              "   {'target_new': 16.005067825317383, 'target_true': 12.133644104003906},\n",
              "   {'target_new': 14.325071334838867, 'target_true': 10.522817611694336},\n",
              "   {'target_new': 16.035396575927734, 'target_true': 12.534464836120605}],\n",
              "  'attribute_prompts_probs': [{'target_new': 15.837298393249512,\n",
              "    'target_true': 12.988123893737793},\n",
              "   {'target_new': 11.54823112487793, 'target_true': 8.751173973083496},\n",
              "   {'target_new': 16.822908401489258, 'target_true': 15.022502899169922},\n",
              "   {'target_new': 14.669233322143555, 'target_true': 13.474388122558594},\n",
              "   {'target_new': 15.890680313110352, 'target_true': 14.438880920410156},\n",
              "   {'target_new': 15.570779800415039, 'target_true': 12.98419189453125},\n",
              "   {'target_new': 10.763813018798828, 'target_true': 8.529622077941895},\n",
              "   {'target_new': 8.171453475952148, 'target_true': 6.908405303955078},\n",
              "   {'target_new': 13.67031192779541, 'target_true': 10.168740272521973},\n",
              "   {'target_new': 14.8511962890625, 'target_true': 10.093777656555176}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.0006288696313276887,\n",
              "    'target_true': 20.27884864807129}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.0711509957909584,\n",
              "    'target_true': 12.50058364868164},\n",
              "   {'target_new': 0.003323628567159176, 'target_true': 16.232147216796875}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 7.461929798126221,\n",
              "    'target_true': 2.855560302734375},\n",
              "   {'target_new': 5.871603012084961, 'target_true': 4.841052055358887},\n",
              "   {'target_new': 7.298947811126709, 'target_true': 6.432887077331543},\n",
              "   {'target_new': 5.8539276123046875, 'target_true': 7.920572280883789},\n",
              "   {'target_new': 6.1298627853393555, 'target_true': 5.335648536682129},\n",
              "   {'target_new': 6.0968337059021, 'target_true': 6.170060157775879},\n",
              "   {'target_new': 7.736902236938477, 'target_true': 3.016796350479126},\n",
              "   {'target_new': 7.649234294891357, 'target_true': 6.446013450622559},\n",
              "   {'target_new': 8.433120727539062, 'target_true': 5.904616355895996},\n",
              "   {'target_new': 8.210409164428711, 'target_true': 3.7164053916931152}],\n",
              "  'attribute_prompts_probs': [{'target_new': 7.483756065368652,\n",
              "    'target_true': 6.608789443969727},\n",
              "   {'target_new': 10.051010131835938, 'target_true': 11.84880256652832},\n",
              "   {'target_new': 8.891791343688965, 'target_true': 9.145723342895508},\n",
              "   {'target_new': 8.394813537597656, 'target_true': 7.363527297973633},\n",
              "   {'target_new': 4.5830607414245605, 'target_true': 9.911998748779297},\n",
              "   {'target_new': 6.501378059387207, 'target_true': 5.92025089263916},\n",
              "   {'target_new': 5.9794745445251465, 'target_true': 3.529794454574585},\n",
              "   {'target_new': 6.716760158538818, 'target_true': 8.386035919189453},\n",
              "   {'target_new': 6.316565036773682, 'target_true': 5.414620399475098},\n",
              "   {'target_new': 4.100202560424805, 'target_true': 8.292832374572754}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.026113135740160942,\n",
              "    'target_true': 15.902127265930176}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.043134190142154694,\n",
              "    'target_true': 15.604246139526367},\n",
              "   {'target_new': 0.2340867519378662, 'target_true': 13.68319320678711}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 14.331120491027832,\n",
              "    'target_true': 3.9499025344848633},\n",
              "   {'target_new': 11.462297439575195, 'target_true': 5.508367538452148},\n",
              "   {'target_new': 10.026227951049805, 'target_true': 6.107879161834717},\n",
              "   {'target_new': 12.355292320251465, 'target_true': 5.055901527404785},\n",
              "   {'target_new': 2.4157896041870117, 'target_true': 7.0010666847229},\n",
              "   {'target_new': 7.423579216003418, 'target_true': 3.4159109592437744},\n",
              "   {'target_new': 12.638486862182617, 'target_true': 3.7812771797180176},\n",
              "   {'target_new': 4.593192100524902, 'target_true': 8.696525573730469},\n",
              "   {'target_new': 11.996273040771484, 'target_true': 9.328743934631348},\n",
              "   {'target_new': 10.820181846618652, 'target_true': 5.625788688659668}],\n",
              "  'attribute_prompts_probs': [{'target_new': 7.117598533630371,\n",
              "    'target_true': 10.48102855682373},\n",
              "   {'target_new': 7.214366912841797, 'target_true': 8.45483684539795},\n",
              "   {'target_new': 1.7136995792388916, 'target_true': 7.04807710647583},\n",
              "   {'target_new': 10.529552459716797, 'target_true': 10.964984893798828},\n",
              "   {'target_new': 5.731106758117676, 'target_true': 9.77323055267334},\n",
              "   {'target_new': 9.116719245910645, 'target_true': 9.630690574645996},\n",
              "   {'target_new': 7.234979629516602, 'target_true': 8.74420166015625},\n",
              "   {'target_new': 8.003805160522461, 'target_true': 7.693431854248047},\n",
              "   {'target_new': 2.1329264640808105, 'target_true': 9.657983779907227},\n",
              "   {'target_new': 4.747247219085693, 'target_true': 11.21895980834961}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.0014262752374634147,\n",
              "    'target_true': 8.434322357177734}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 6.095250129699707,\n",
              "    'target_true': 10.82652759552002},\n",
              "   {'target_new': 0.12531259655952454, 'target_true': 7.519562244415283}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 4.594021797180176,\n",
              "    'target_true': 3.263152599334717},\n",
              "   {'target_new': 4.77437686920166, 'target_true': 5.5505170822143555},\n",
              "   {'target_new': 8.917648315429688, 'target_true': 8.608136177062988},\n",
              "   {'target_new': 2.46161150932312, 'target_true': 4.134443283081055},\n",
              "   {'target_new': 2.189544677734375, 'target_true': 0.7159738540649414},\n",
              "   {'target_new': 7.141190528869629, 'target_true': 5.685502052307129},\n",
              "   {'target_new': 6.559802532196045, 'target_true': 8.234834671020508},\n",
              "   {'target_new': 7.545767784118652, 'target_true': 4.447849273681641},\n",
              "   {'target_new': 6.895430564880371, 'target_true': 7.630013942718506},\n",
              "   {'target_new': 6.883831024169922, 'target_true': 5.355686187744141}],\n",
              "  'attribute_prompts_probs': [{'target_new': 6.073073387145996,\n",
              "    'target_true': 8.18818187713623},\n",
              "   {'target_new': 2.3669168949127197, 'target_true': 5.416706562042236},\n",
              "   {'target_new': 4.537675380706787, 'target_true': 6.249975681304932},\n",
              "   {'target_new': 9.555710792541504, 'target_true': 10.30005931854248},\n",
              "   {'target_new': 5.9264984130859375, 'target_true': 10.097288131713867},\n",
              "   {'target_new': 8.766910552978516, 'target_true': 10.399656295776367},\n",
              "   {'target_new': 2.1054186820983887, 'target_true': 3.485325336456299},\n",
              "   {'target_new': 3.4170494079589844, 'target_true': 7.830353260040283},\n",
              "   {'target_new': 3.689558267593384, 'target_true': 7.417482376098633},\n",
              "   {'target_new': 3.5576024055480957, 'target_true': 5.819618225097656}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.005600357428193092,\n",
              "    'target_true': 10.44565200805664}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.047688018530607224,\n",
              "    'target_true': 5.820707321166992},\n",
              "   {'target_new': 0.10458274185657501, 'target_true': 4.498437881469727}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 5.833148002624512,\n",
              "    'target_true': 1.2983200550079346},\n",
              "   {'target_new': 5.739151954650879, 'target_true': 0.5579805374145508},\n",
              "   {'target_new': 7.328366756439209, 'target_true': 2.8447465896606445},\n",
              "   {'target_new': 4.112551689147949, 'target_true': 0.2988491356372833},\n",
              "   {'target_new': 7.519604682922363, 'target_true': 0.8653287887573242},\n",
              "   {'target_new': 10.458341598510742, 'target_true': 8.70168399810791},\n",
              "   {'target_new': 11.0581693649292, 'target_true': 6.504947185516357},\n",
              "   {'target_new': 9.455077171325684, 'target_true': 7.052816867828369},\n",
              "   {'target_new': 7.687073230743408, 'target_true': 2.532742738723755},\n",
              "   {'target_new': 10.880109786987305, 'target_true': 7.6380615234375}],\n",
              "  'attribute_prompts_probs': [{'target_new': 9.26035213470459,\n",
              "    'target_true': 11.512843132019043},\n",
              "   {'target_new': 6.415637016296387, 'target_true': 10.166099548339844},\n",
              "   {'target_new': 2.2182650566101074, 'target_true': 7.4434051513671875},\n",
              "   {'target_new': 4.707518100738525, 'target_true': 9.766584396362305},\n",
              "   {'target_new': 1.022658109664917, 'target_true': 7.584440231323242},\n",
              "   {'target_new': 0.19879674911499023, 'target_true': 7.989482402801514},\n",
              "   {'target_new': 0.3721538782119751, 'target_true': 6.153423309326172},\n",
              "   {'target_new': 0.35960516333580017, 'target_true': 6.113773345947266},\n",
              "   {'target_new': 0.20939630270004272, 'target_true': 6.465442180633545},\n",
              "   {'target_new': 2.732248544692993, 'target_true': 11.409536361694336}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.004079235717654228,\n",
              "    'target_true': 21.54844856262207}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.008054514415562153,\n",
              "    'target_true': 19.09165382385254},\n",
              "   {'target_new': 0.039750099182128906, 'target_true': 17.865886688232422}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 7.962264060974121,\n",
              "    'target_true': 3.013937473297119},\n",
              "   {'target_new': 0.019165102392435074, 'target_true': 15.1862211227417},\n",
              "   {'target_new': 9.552584648132324, 'target_true': 4.2599053382873535},\n",
              "   {'target_new': 9.642271041870117, 'target_true': 1.0592187643051147},\n",
              "   {'target_new': 9.60596752166748, 'target_true': 1.5177839994430542},\n",
              "   {'target_new': 5.413098335266113, 'target_true': 1.7471281290054321},\n",
              "   {'target_new': 9.275961875915527, 'target_true': 3.03887939453125},\n",
              "   {'target_new': 3.3048324584960938, 'target_true': 2.9837732315063477},\n",
              "   {'target_new': 8.096077919006348, 'target_true': 1.37466299533844},\n",
              "   {'target_new': 0.1229436993598938, 'target_true': 15.874123573303223}],\n",
              "  'attribute_prompts_probs': [{'target_new': 0.13936710357666016,\n",
              "    'target_true': 14.608489036560059},\n",
              "   {'target_new': 3.3065366744995117, 'target_true': 9.42654037475586},\n",
              "   {'target_new': 1.018070936203003, 'target_true': 11.487482070922852},\n",
              "   {'target_new': 1.4256409406661987, 'target_true': 12.802165031433105},\n",
              "   {'target_new': 3.7182912826538086, 'target_true': 9.071918487548828},\n",
              "   {'target_new': 1.4627336263656616, 'target_true': 11.353164672851562},\n",
              "   {'target_new': 0.17216773331165314, 'target_true': 13.547887802124023},\n",
              "   {'target_new': 0.4679475426673889, 'target_true': 14.184099197387695},\n",
              "   {'target_new': 0.079041987657547, 'target_true': 15.59400463104248},\n",
              "   {'target_new': 1.0853004455566406, 'target_true': 13.731019973754883}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.008431551977992058,\n",
              "    'target_true': 15.494795799255371}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 3.61484956741333,\n",
              "    'target_true': 9.892075538635254},\n",
              "   {'target_new': 2.634535074234009, 'target_true': 16.4533634185791}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 9.693265914916992,\n",
              "    'target_true': 4.971902847290039},\n",
              "   {'target_new': 5.490697383880615, 'target_true': 6.66077184677124},\n",
              "   {'target_new': 1.5504435300827026, 'target_true': 2.270688056945801},\n",
              "   {'target_new': 7.660529136657715, 'target_true': 2.3151450157165527},\n",
              "   {'target_new': 8.06193733215332, 'target_true': 5.530123233795166},\n",
              "   {'target_new': 9.317167282104492, 'target_true': 10.838720321655273},\n",
              "   {'target_new': 5.524398326873779, 'target_true': 0.4234427809715271},\n",
              "   {'target_new': 9.72645378112793, 'target_true': 7.77467155456543},\n",
              "   {'target_new': 4.473205089569092, 'target_true': 0.9996559023857117},\n",
              "   {'target_new': 0.5412272214889526, 'target_true': 4.812672138214111}],\n",
              "  'attribute_prompts_probs': [{'target_new': 0.7273045778274536,\n",
              "    'target_true': 7.051141262054443},\n",
              "   {'target_new': 2.023989200592041, 'target_true': 10.538446426391602},\n",
              "   {'target_new': 4.341436862945557, 'target_true': 14.631487846374512},\n",
              "   {'target_new': 7.391937255859375, 'target_true': 12.634160995483398},\n",
              "   {'target_new': 1.3908753395080566, 'target_true': 10.788103103637695},\n",
              "   {'target_new': 0.5291703343391418, 'target_true': 7.575806140899658},\n",
              "   {'target_new': 7.861117362976074, 'target_true': 11.89064884185791},\n",
              "   {'target_new': 3.4639456272125244, 'target_true': 9.696702003479004},\n",
              "   {'target_new': 9.943910598754883, 'target_true': 16.53476333618164},\n",
              "   {'target_new': 0.2988499104976654, 'target_true': 10.450879096984863}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.11470524966716766,\n",
              "    'target_true': 19.204492568969727}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 4.176167964935303,\n",
              "    'target_true': 12.273294448852539},\n",
              "   {'target_new': 0.2612617611885071, 'target_true': 15.09018611907959}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 6.566248416900635,\n",
              "    'target_true': 5.600471019744873},\n",
              "   {'target_new': 8.135629653930664, 'target_true': 8.178083419799805},\n",
              "   {'target_new': 12.145410537719727, 'target_true': 5.339097023010254},\n",
              "   {'target_new': 12.17503833770752, 'target_true': 8.001866340637207},\n",
              "   {'target_new': 6.545347690582275, 'target_true': 4.9426679611206055},\n",
              "   {'target_new': 11.116488456726074, 'target_true': 6.429477691650391},\n",
              "   {'target_new': 12.976080894470215, 'target_true': 6.091660976409912},\n",
              "   {'target_new': 15.048994064331055, 'target_true': 9.71969223022461},\n",
              "   {'target_new': 11.856515884399414, 'target_true': 7.516792297363281},\n",
              "   {'target_new': 16.289688110351562, 'target_true': 4.869208335876465}],\n",
              "  'attribute_prompts_probs': [{'target_new': 3.4272704124450684,\n",
              "    'target_true': 13.245317459106445},\n",
              "   {'target_new': 7.083536624908447, 'target_true': 8.749456405639648},\n",
              "   {'target_new': 15.39319133758545, 'target_true': 13.756392478942871},\n",
              "   {'target_new': 16.747787475585938, 'target_true': 14.91275405883789},\n",
              "   {'target_new': 8.71812629699707, 'target_true': 10.79196548461914},\n",
              "   {'target_new': 14.189624786376953, 'target_true': 9.715306282043457},\n",
              "   {'target_new': 7.584136009216309, 'target_true': 10.985906600952148},\n",
              "   {'target_new': 16.30124282836914, 'target_true': 13.958944320678711},\n",
              "   {'target_new': 6.104301929473877, 'target_true': 8.601940155029297},\n",
              "   {'target_new': 4.6207122802734375, 'target_true': 10.740163803100586}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.008765434846282005,\n",
              "    'target_true': 13.989618301391602}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 8.083161354064941,\n",
              "    'target_true': 11.281513214111328},\n",
              "   {'target_new': 5.072685718536377, 'target_true': 11.539966583251953}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 13.395082473754883,\n",
              "    'target_true': 13.013648986816406},\n",
              "   {'target_new': 10.650178909301758, 'target_true': 11.811675071716309},\n",
              "   {'target_new': 14.929083824157715, 'target_true': 13.36041259765625},\n",
              "   {'target_new': 8.556710243225098, 'target_true': 7.542217254638672},\n",
              "   {'target_new': 11.513628005981445, 'target_true': 13.239361763000488},\n",
              "   {'target_new': 13.514466285705566, 'target_true': 14.523794174194336},\n",
              "   {'target_new': 11.14314079284668, 'target_true': 10.52719783782959},\n",
              "   {'target_new': 13.991632461547852, 'target_true': 12.891435623168945},\n",
              "   {'target_new': 16.817846298217773, 'target_true': 14.870030403137207},\n",
              "   {'target_new': 13.43371868133545, 'target_true': 12.96045970916748}],\n",
              "  'attribute_prompts_probs': [{'target_new': 10.532042503356934,\n",
              "    'target_true': 14.599212646484375},\n",
              "   {'target_new': 6.097161293029785, 'target_true': 10.110159873962402},\n",
              "   {'target_new': 10.312315940856934, 'target_true': 10.777772903442383},\n",
              "   {'target_new': 16.426259994506836, 'target_true': 15.138284683227539},\n",
              "   {'target_new': 12.064372062683105, 'target_true': 10.87070083618164},\n",
              "   {'target_new': 17.66160011291504, 'target_true': 16.64670753479004},\n",
              "   {'target_new': 12.611838340759277, 'target_true': 16.376178741455078},\n",
              "   {'target_new': 10.123845100402832, 'target_true': 13.216389656066895},\n",
              "   {'target_new': 12.492942810058594, 'target_true': 15.5341157913208},\n",
              "   {'target_new': 10.124214172363281, 'target_true': 14.715682983398438}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.06000319495797157,\n",
              "    'target_true': 14.906036376953125}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.23460429906845093,\n",
              "    'target_true': 13.691040992736816},\n",
              "   {'target_new': 0.14027543365955353, 'target_true': 13.120065689086914}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 11.465679168701172,\n",
              "    'target_true': 7.75974702835083},\n",
              "   {'target_new': 10.929182052612305, 'target_true': 6.271236896514893},\n",
              "   {'target_new': 11.207815170288086, 'target_true': 7.482580184936523},\n",
              "   {'target_new': 10.344862937927246, 'target_true': 5.846261978149414},\n",
              "   {'target_new': 11.485372543334961, 'target_true': 7.610198020935059},\n",
              "   {'target_new': 10.725772857666016, 'target_true': 8.854230880737305},\n",
              "   {'target_new': 11.10069465637207, 'target_true': 6.218695640563965},\n",
              "   {'target_new': 10.954132080078125, 'target_true': 7.226391792297363},\n",
              "   {'target_new': 10.67864990234375, 'target_true': 6.323246955871582},\n",
              "   {'target_new': 11.476874351501465, 'target_true': 9.928791046142578}],\n",
              "  'attribute_prompts_probs': [{'target_new': 10.365255355834961,\n",
              "    'target_true': 10.914365768432617},\n",
              "   {'target_new': 12.979012489318848, 'target_true': 12.039031028747559},\n",
              "   {'target_new': 5.928936004638672, 'target_true': 10.266597747802734},\n",
              "   {'target_new': 4.396097183227539, 'target_true': 9.175052642822266},\n",
              "   {'target_new': 6.177822113037109, 'target_true': 12.440357208251953},\n",
              "   {'target_new': 9.927346229553223, 'target_true': 8.210714340209961},\n",
              "   {'target_new': 7.1724748611450195, 'target_true': 9.658350944519043},\n",
              "   {'target_new': 11.07796859741211, 'target_true': 11.22476863861084},\n",
              "   {'target_new': 10.18425178527832, 'target_true': 8.611278533935547},\n",
              "   {'target_new': 6.617730617523193, 'target_true': 13.121578216552734}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.05439741536974907,\n",
              "    'target_true': 8.106155395507812}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.16345612704753876,\n",
              "    'target_true': 6.734583854675293},\n",
              "   {'target_new': 0.06582431495189667, 'target_true': 8.205083847045898}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 6.348663330078125,\n",
              "    'target_true': 1.0437642335891724},\n",
              "   {'target_new': 3.4138379096984863, 'target_true': 2.3242287635803223},\n",
              "   {'target_new': 0.22189565002918243, 'target_true': 6.0694427490234375},\n",
              "   {'target_new': 6.068305969238281, 'target_true': 1.8774980306625366},\n",
              "   {'target_new': 4.9052734375, 'target_true': 1.8929295539855957},\n",
              "   {'target_new': 5.776891231536865, 'target_true': 2.6643195152282715},\n",
              "   {'target_new': 4.004921913146973, 'target_true': 3.584066867828369},\n",
              "   {'target_new': 5.288448333740234, 'target_true': 3.2786521911621094},\n",
              "   {'target_new': 3.4440951347351074, 'target_true': 5.104373931884766},\n",
              "   {'target_new': 6.625904083251953, 'target_true': 1.407526969909668}],\n",
              "  'attribute_prompts_probs': [{'target_new': 1.2443759441375732,\n",
              "    'target_true': 10.378064155578613},\n",
              "   {'target_new': 0.25769510865211487, 'target_true': 15.561558723449707},\n",
              "   {'target_new': 0.4609168767929077, 'target_true': 14.766251564025879},\n",
              "   {'target_new': 0.4219147264957428, 'target_true': 13.134124755859375},\n",
              "   {'target_new': 1.1403864622116089, 'target_true': 10.678749084472656},\n",
              "   {'target_new': 0.10178501904010773, 'target_true': 10.262333869934082},\n",
              "   {'target_new': 0.4123239517211914, 'target_true': 8.49733829498291},\n",
              "   {'target_new': 1.7710390090942383, 'target_true': 8.837757110595703},\n",
              "   {'target_new': 0.43074682354927063, 'target_true': 9.138572692871094},\n",
              "   {'target_new': 0.8516120314598083, 'target_true': 8.941832542419434}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.09458037465810776,\n",
              "    'target_true': 7.615428447723389}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.5159385204315186,\n",
              "    'target_true': 8.411642074584961},\n",
              "   {'target_new': 0.6706842184066772, 'target_true': 6.859477996826172}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 14.091633796691895,\n",
              "    'target_true': 12.796311378479004},\n",
              "   {'target_new': 9.394639015197754, 'target_true': 5.588001728057861},\n",
              "   {'target_new': 8.391796112060547, 'target_true': 7.796063423156738},\n",
              "   {'target_new': 10.179519653320312, 'target_true': 9.46420955657959},\n",
              "   {'target_new': 15.386743545532227, 'target_true': 13.627405166625977},\n",
              "   {'target_new': 4.705708980560303, 'target_true': 4.489474773406982},\n",
              "   {'target_new': 14.1427583694458, 'target_true': 12.785088539123535},\n",
              "   {'target_new': 7.3678131103515625, 'target_true': 5.9327921867370605},\n",
              "   {'target_new': 6.811760902404785, 'target_true': 7.3839216232299805},\n",
              "   {'target_new': 7.277881622314453, 'target_true': 5.446384429931641}],\n",
              "  'attribute_prompts_probs': [{'target_new': 15.893811225891113,\n",
              "    'target_true': 14.758987426757812},\n",
              "   {'target_new': 13.539021492004395, 'target_true': 13.676348686218262},\n",
              "   {'target_new': 6.931581497192383, 'target_true': 6.840669631958008},\n",
              "   {'target_new': 3.1955227851867676, 'target_true': 4.969582557678223},\n",
              "   {'target_new': 3.240265369415283, 'target_true': 5.3887553215026855},\n",
              "   {'target_new': 4.854983806610107, 'target_true': 6.38954496383667},\n",
              "   {'target_new': 7.596034049987793, 'target_true': 6.997138977050781},\n",
              "   {'target_new': 13.690078735351562, 'target_true': 12.056246757507324},\n",
              "   {'target_new': 3.7160651683807373, 'target_true': 3.8971915245056152},\n",
              "   {'target_new': 3.3141067028045654, 'target_true': 4.490748405456543}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.0076158177107572556,\n",
              "    'target_true': 11.117193222045898}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.5770644545555115,\n",
              "    'target_true': 5.970678806304932},\n",
              "   {'target_new': 0.022359415888786316, 'target_true': 10.7879638671875}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 5.399796962738037,\n",
              "    'target_true': 2.0566225051879883},\n",
              "   {'target_new': 7.068119049072266, 'target_true': 2.353170394897461},\n",
              "   {'target_new': 10.951181411743164, 'target_true': 9.70673656463623},\n",
              "   {'target_new': 6.9399800300598145, 'target_true': 3.3540544509887695},\n",
              "   {'target_new': 7.714529991149902, 'target_true': 3.4715864658355713},\n",
              "   {'target_new': 9.364452362060547, 'target_true': 7.159337043762207},\n",
              "   {'target_new': 5.169447898864746, 'target_true': 0.6709369421005249},\n",
              "   {'target_new': 3.398496389389038, 'target_true': 3.317518711090088},\n",
              "   {'target_new': 7.412285327911377, 'target_true': 3.1491751670837402},\n",
              "   {'target_new': 7.257685661315918, 'target_true': 2.0930709838867188}],\n",
              "  'attribute_prompts_probs': [{'target_new': 9.765068054199219,\n",
              "    'target_true': 15.282011032104492},\n",
              "   {'target_new': 9.836925506591797, 'target_true': 12.193331718444824},\n",
              "   {'target_new': 9.438145637512207, 'target_true': 8.891996383666992},\n",
              "   {'target_new': 3.7752346992492676, 'target_true': 10.509682655334473},\n",
              "   {'target_new': 3.5393266677856445, 'target_true': 3.0854363441467285},\n",
              "   {'target_new': 4.2461442947387695, 'target_true': 5.881034851074219},\n",
              "   {'target_new': 2.4224276542663574, 'target_true': 8.758468627929688},\n",
              "   {'target_new': 9.47912311553955, 'target_true': 11.817370414733887},\n",
              "   {'target_new': 9.638959884643555, 'target_true': 12.482376098632812},\n",
              "   {'target_new': 5.051840305328369, 'target_true': 9.426815032958984}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.020200245082378387,\n",
              "    'target_true': 13.851348876953125}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.055012404918670654,\n",
              "    'target_true': 11.685813903808594},\n",
              "   {'target_new': 0.14183101058006287, 'target_true': 13.790802001953125}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 0.4107683002948761,\n",
              "    'target_true': 5.421880722045898},\n",
              "   {'target_new': 2.1126327514648438, 'target_true': 3.6968131065368652},\n",
              "   {'target_new': 3.631955623626709, 'target_true': 4.791931629180908},\n",
              "   {'target_new': 7.589053153991699, 'target_true': 3.1053788661956787},\n",
              "   {'target_new': 2.6178319454193115, 'target_true': 5.741954803466797},\n",
              "   {'target_new': 8.91412353515625, 'target_true': 1.6697807312011719},\n",
              "   {'target_new': 2.0555429458618164, 'target_true': 4.079411506652832},\n",
              "   {'target_new': 1.5256351232528687, 'target_true': 3.2167959213256836},\n",
              "   {'target_new': 12.8904390335083, 'target_true': 6.164799690246582},\n",
              "   {'target_new': 4.631165027618408, 'target_true': 5.225048065185547}],\n",
              "  'attribute_prompts_probs': [{'target_new': 3.600752115249634,\n",
              "    'target_true': 9.78197193145752},\n",
              "   {'target_new': 1.8598203659057617, 'target_true': 7.936796188354492},\n",
              "   {'target_new': 1.8681776523590088, 'target_true': 8.06268310546875},\n",
              "   {'target_new': 5.480294227600098, 'target_true': 10.680198669433594},\n",
              "   {'target_new': 5.167052268981934, 'target_true': 6.013952732086182},\n",
              "   {'target_new': 5.704466819763184, 'target_true': 10.142165184020996},\n",
              "   {'target_new': 4.875675201416016, 'target_true': 9.308309555053711},\n",
              "   {'target_new': 2.5917391777038574, 'target_true': 6.967875003814697},\n",
              "   {'target_new': 5.4186835289001465, 'target_true': 10.129960060119629},\n",
              "   {'target_new': 2.7990987300872803, 'target_true': 6.27358341217041}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.014475567266345024,\n",
              "    'target_true': 16.516359329223633}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.09351861476898193,\n",
              "    'target_true': 11.806653022766113},\n",
              "   {'target_new': 0.08767961710691452, 'target_true': 12.183574676513672}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 7.838878154754639,\n",
              "    'target_true': 7.394589424133301},\n",
              "   {'target_new': 8.79231071472168, 'target_true': 7.041898727416992},\n",
              "   {'target_new': 7.3593339920043945, 'target_true': 5.324649810791016},\n",
              "   {'target_new': 8.061527252197266, 'target_true': 6.2974958419799805},\n",
              "   {'target_new': 7.537425518035889, 'target_true': 4.920336723327637},\n",
              "   {'target_new': 9.552248001098633, 'target_true': 5.6986212730407715},\n",
              "   {'target_new': 6.005618095397949, 'target_true': 6.164797782897949},\n",
              "   {'target_new': 7.660794258117676, 'target_true': 6.332291126251221},\n",
              "   {'target_new': 9.987263679504395, 'target_true': 8.69764518737793},\n",
              "   {'target_new': 7.575008392333984, 'target_true': 6.24859619140625}],\n",
              "  'attribute_prompts_probs': [{'target_new': 4.917724609375,\n",
              "    'target_true': 7.226813316345215},\n",
              "   {'target_new': 1.3494514226913452, 'target_true': 11.147241592407227},\n",
              "   {'target_new': 5.659856796264648, 'target_true': 6.167967796325684},\n",
              "   {'target_new': 7.088281631469727, 'target_true': 8.272904396057129},\n",
              "   {'target_new': 3.3842129707336426, 'target_true': 9.502440452575684},\n",
              "   {'target_new': 6.143405914306641, 'target_true': 6.048282623291016},\n",
              "   {'target_new': 2.2732579708099365, 'target_true': 10.565131187438965},\n",
              "   {'target_new': 7.236268520355225, 'target_true': 7.107726097106934},\n",
              "   {'target_new': 0.6142168641090393, 'target_true': 11.036014556884766},\n",
              "   {'target_new': 2.97575044631958, 'target_true': 9.363798141479492}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.0034538398031145334,\n",
              "    'target_true': 19.91276741027832}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.3666892349720001,\n",
              "    'target_true': 15.117948532104492},\n",
              "   {'target_new': 4.5256757736206055, 'target_true': 14.432953834533691}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 15.405813217163086,\n",
              "    'target_true': 14.79020881652832},\n",
              "   {'target_new': 10.866398811340332, 'target_true': 2.943868398666382},\n",
              "   {'target_new': 8.879404067993164, 'target_true': 9.11473274230957},\n",
              "   {'target_new': 10.564102172851562, 'target_true': 13.816994667053223},\n",
              "   {'target_new': 10.685908317565918, 'target_true': 9.599802017211914},\n",
              "   {'target_new': 9.535795211791992, 'target_true': 10.43740463256836},\n",
              "   {'target_new': 6.261534690856934, 'target_true': 8.03943157196045},\n",
              "   {'target_new': 9.399514198303223, 'target_true': 8.794939041137695},\n",
              "   {'target_new': 8.3367338180542, 'target_true': 11.316964149475098},\n",
              "   {'target_new': 10.534276008605957, 'target_true': 11.203130722045898}],\n",
              "  'attribute_prompts_probs': [{'target_new': 6.840867042541504,\n",
              "    'target_true': 10.198963165283203},\n",
              "   {'target_new': 7.434206008911133, 'target_true': 12.019266128540039},\n",
              "   {'target_new': 8.415362358093262, 'target_true': 12.08874797821045},\n",
              "   {'target_new': 6.176733493804932, 'target_true': 12.080718040466309},\n",
              "   {'target_new': 3.4995713233947754, 'target_true': 10.023646354675293},\n",
              "   {'target_new': 11.504358291625977, 'target_true': 12.145271301269531},\n",
              "   {'target_new': 6.165721893310547, 'target_true': 10.419931411743164},\n",
              "   {'target_new': 4.486058235168457, 'target_true': 10.212089538574219},\n",
              "   {'target_new': 6.03410530090332, 'target_true': 10.211090087890625},\n",
              "   {'target_new': 7.7262444496154785, 'target_true': 12.029915809631348}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.014764800667762756,\n",
              "    'target_true': 12.894510269165039}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 6.7868804931640625,\n",
              "    'target_true': 7.320265769958496},\n",
              "   {'target_new': 6.769722938537598, 'target_true': 10.115466117858887}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 10.01209831237793,\n",
              "    'target_true': 7.135507106781006},\n",
              "   {'target_new': 8.546015739440918, 'target_true': 7.2842206954956055},\n",
              "   {'target_new': 7.392004013061523, 'target_true': 5.8892107009887695},\n",
              "   {'target_new': 11.731210708618164, 'target_true': 11.48263168334961},\n",
              "   {'target_new': 12.911497116088867, 'target_true': 12.31656265258789},\n",
              "   {'target_new': 12.618494033813477, 'target_true': 8.518326759338379},\n",
              "   {'target_new': 12.16094970703125, 'target_true': 9.30410385131836},\n",
              "   {'target_new': 10.077020645141602, 'target_true': 8.17740249633789},\n",
              "   {'target_new': 10.465720176696777, 'target_true': 9.30253791809082},\n",
              "   {'target_new': 14.675261497497559, 'target_true': 12.829571723937988}],\n",
              "  'attribute_prompts_probs': [{'target_new': 10.478521347045898,\n",
              "    'target_true': 10.522963523864746},\n",
              "   {'target_new': 9.724418640136719, 'target_true': 8.714719772338867},\n",
              "   {'target_new': 7.260776519775391, 'target_true': 5.8047895431518555},\n",
              "   {'target_new': 10.025697708129883, 'target_true': 9.499410629272461},\n",
              "   {'target_new': 11.825770378112793, 'target_true': 7.26340389251709},\n",
              "   {'target_new': 6.469476222991943, 'target_true': 5.171581268310547},\n",
              "   {'target_new': 8.970141410827637, 'target_true': 8.726851463317871},\n",
              "   {'target_new': 6.2709269523620605, 'target_true': 5.911115646362305},\n",
              "   {'target_new': 10.487021446228027, 'target_true': 9.900035858154297},\n",
              "   {'target_new': 8.85881519317627, 'target_true': 9.83761215209961}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.10112573206424713,\n",
              "    'target_true': 9.622604370117188}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.1509452611207962,\n",
              "    'target_true': 9.099900245666504},\n",
              "   {'target_new': 0.0872725397348404, 'target_true': 7.435976028442383}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 12.227941513061523,\n",
              "    'target_true': 1.1121206283569336},\n",
              "   {'target_new': 14.529369354248047, 'target_true': 0.4486202597618103},\n",
              "   {'target_new': 15.464042663574219, 'target_true': 0.27415749430656433},\n",
              "   {'target_new': 11.551015853881836, 'target_true': 2.05505633354187},\n",
              "   {'target_new': 16.369413375854492, 'target_true': 0.7836300134658813},\n",
              "   {'target_new': 11.924276351928711, 'target_true': 2.7469522953033447},\n",
              "   {'target_new': 15.966211318969727, 'target_true': 0.18507294356822968},\n",
              "   {'target_new': 11.643793106079102, 'target_true': 2.303273916244507},\n",
              "   {'target_new': 15.097976684570312, 'target_true': 0.20007789134979248},\n",
              "   {'target_new': 13.287287712097168, 'target_true': 1.887359857559204}],\n",
              "  'attribute_prompts_probs': [{'target_new': 6.609166622161865,\n",
              "    'target_true': 10.345221519470215},\n",
              "   {'target_new': 2.9188265800476074, 'target_true': 11.905045509338379},\n",
              "   {'target_new': 5.390595436096191, 'target_true': 12.628599166870117},\n",
              "   {'target_new': 2.951313018798828, 'target_true': 10.83523178100586},\n",
              "   {'target_new': 11.292284965515137, 'target_true': 7.955080032348633},\n",
              "   {'target_new': 13.100724220275879, 'target_true': 7.819854736328125},\n",
              "   {'target_new': 12.439468383789062, 'target_true': 7.699242115020752},\n",
              "   {'target_new': 3.357820987701416, 'target_true': 10.953747749328613},\n",
              "   {'target_new': 2.3288116455078125, 'target_true': 9.763772964477539},\n",
              "   {'target_new': 4.092658996582031, 'target_true': 10.11932373046875}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.030615821480751038,\n",
              "    'target_true': 16.749313354492188}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 10.383842468261719,\n",
              "    'target_true': 15.032668113708496},\n",
              "   {'target_new': 9.085803985595703, 'target_true': 13.892341613769531}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 14.073322296142578,\n",
              "    'target_true': 13.368242263793945},\n",
              "   {'target_new': 12.617427825927734, 'target_true': 13.234025955200195},\n",
              "   {'target_new': 12.755334854125977, 'target_true': 12.76220703125},\n",
              "   {'target_new': 14.938497543334961, 'target_true': 12.616991996765137},\n",
              "   {'target_new': 12.961631774902344, 'target_true': 11.586965560913086},\n",
              "   {'target_new': 12.095197677612305, 'target_true': 8.755800247192383},\n",
              "   {'target_new': 13.7118558883667, 'target_true': 12.952566146850586},\n",
              "   {'target_new': 11.280572891235352, 'target_true': 9.362024307250977},\n",
              "   {'target_new': 10.108634948730469, 'target_true': 7.346724510192871},\n",
              "   {'target_new': 10.9804048538208, 'target_true': 10.142437934875488}],\n",
              "  'attribute_prompts_probs': [{'target_new': 12.365801811218262,\n",
              "    'target_true': 14.389376640319824},\n",
              "   {'target_new': 6.1534423828125, 'target_true': 5.896513938903809},\n",
              "   {'target_new': 14.119600296020508, 'target_true': 14.647012710571289},\n",
              "   {'target_new': 6.4431376457214355, 'target_true': 12.1450777053833},\n",
              "   {'target_new': 11.586957931518555, 'target_true': 14.783468246459961},\n",
              "   {'target_new': 9.882452011108398, 'target_true': 11.970008850097656},\n",
              "   {'target_new': 10.34554672241211, 'target_true': 12.662960052490234},\n",
              "   {'target_new': 3.7659213542938232, 'target_true': 11.890984535217285},\n",
              "   {'target_new': 4.662140846252441, 'target_true': 2.8022541999816895},\n",
              "   {'target_new': 4.113411903381348, 'target_true': 4.0465850830078125}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.012716514989733696,\n",
              "    'target_true': 11.446521759033203}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.17247004806995392,\n",
              "    'target_true': 6.4444122314453125},\n",
              "   {'target_new': 3.2368175983428955, 'target_true': 8.078293800354004}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 8.706905364990234,\n",
              "    'target_true': 0.13192975521087646},\n",
              "   {'target_new': 8.785587310791016, 'target_true': 4.204737663269043},\n",
              "   {'target_new': 9.06258487701416, 'target_true': 8.414127349853516},\n",
              "   {'target_new': 6.926121711730957, 'target_true': 0.3392712473869324},\n",
              "   {'target_new': 7.99943208694458, 'target_true': 6.471994400024414},\n",
              "   {'target_new': 10.884350776672363, 'target_true': 7.892572402954102},\n",
              "   {'target_new': 6.155769348144531, 'target_true': 0.34930288791656494},\n",
              "   {'target_new': 7.519604682922363, 'target_true': 0.8653287887573242},\n",
              "   {'target_new': 6.59407901763916, 'target_true': 0.32493147253990173},\n",
              "   {'target_new': 7.486902713775635, 'target_true': 3.0211899280548096}],\n",
              "  'attribute_prompts_probs': [{'target_new': 7.523214340209961,\n",
              "    'target_true': 9.672337532043457},\n",
              "   {'target_new': 7.126677513122559, 'target_true': 9.948554992675781},\n",
              "   {'target_new': 2.977004051208496, 'target_true': 6.261546611785889},\n",
              "   {'target_new': 1.8053693771362305, 'target_true': 6.9879841804504395},\n",
              "   {'target_new': 0.3026888072490692, 'target_true': 5.719113826751709},\n",
              "   {'target_new': 2.337411642074585, 'target_true': 7.539643287658691},\n",
              "   {'target_new': 3.5716731548309326, 'target_true': 11.284017562866211},\n",
              "   {'target_new': 0.2648274600505829, 'target_true': 4.8759050369262695},\n",
              "   {'target_new': 2.847623109817505, 'target_true': 6.548544406890869},\n",
              "   {'target_new': 0.45918959379196167, 'target_true': 6.176126480102539}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.005484296474605799,\n",
              "    'target_true': 12.32261848449707}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 2.50663161277771,\n",
              "    'target_true': 11.043468475341797},\n",
              "   {'target_new': 2.055281162261963, 'target_true': 10.288508415222168}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 6.841573715209961,\n",
              "    'target_true': 6.993656635284424},\n",
              "   {'target_new': 3.891831874847412, 'target_true': 10.797428131103516},\n",
              "   {'target_new': 0.019578073173761368, 'target_true': 10.314635276794434},\n",
              "   {'target_new': 7.35318660736084, 'target_true': 4.552572727203369},\n",
              "   {'target_new': 0.015458408743143082, 'target_true': 11.431666374206543},\n",
              "   {'target_new': 4.189731121063232, 'target_true': 13.635436058044434},\n",
              "   {'target_new': 7.696331977844238, 'target_true': 8.095672607421875},\n",
              "   {'target_new': 4.951498031616211, 'target_true': 0.4062826335430145},\n",
              "   {'target_new': 9.545528411865234, 'target_true': 8.923233985900879},\n",
              "   {'target_new': 3.698406219482422, 'target_true': 10.67094898223877}],\n",
              "  'attribute_prompts_probs': [{'target_new': 0.2196369469165802,\n",
              "    'target_true': 9.053173065185547},\n",
              "   {'target_new': 1.802276372909546, 'target_true': 9.939210891723633},\n",
              "   {'target_new': 9.190286636352539, 'target_true': 11.164876937866211},\n",
              "   {'target_new': 0.23730556666851044, 'target_true': 8.07160758972168},\n",
              "   {'target_new': 4.413028240203857, 'target_true': 10.146245956420898},\n",
              "   {'target_new': 5.884510040283203, 'target_true': 11.935830116271973},\n",
              "   {'target_new': 5.596980094909668, 'target_true': 10.897977828979492},\n",
              "   {'target_new': 5.089382171630859, 'target_true': 9.697684288024902},\n",
              "   {'target_new': 0.2746039927005768, 'target_true': 8.734206199645996},\n",
              "   {'target_new': 4.6061110496521, 'target_true': 7.959295749664307}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.0005514293443411589,\n",
              "    'target_true': 13.2838134765625}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.0009254701435565948,\n",
              "    'target_true': 13.016595840454102},\n",
              "   {'target_new': 4.127893447875977, 'target_true': 9.702524185180664}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 15.5863676071167,\n",
              "    'target_true': 8.837306022644043},\n",
              "   {'target_new': 10.84399127960205, 'target_true': 7.52345085144043},\n",
              "   {'target_new': 5.490384101867676, 'target_true': 1.9597185850143433},\n",
              "   {'target_new': 6.486709117889404, 'target_true': 0.817643940448761},\n",
              "   {'target_new': 10.019829750061035, 'target_true': 7.227659225463867},\n",
              "   {'target_new': 8.5643310546875, 'target_true': 6.937952995300293},\n",
              "   {'target_new': 11.015336036682129, 'target_true': 8.365262031555176},\n",
              "   {'target_new': 3.518993377685547, 'target_true': 1.6486586332321167},\n",
              "   {'target_new': 8.81495475769043, 'target_true': 7.404057025909424},\n",
              "   {'target_new': 9.11298942565918, 'target_true': 7.03253173828125}],\n",
              "  'attribute_prompts_probs': [{'target_new': 5.749715328216553,\n",
              "    'target_true': 6.899092197418213},\n",
              "   {'target_new': 0.6697235703468323, 'target_true': 1.9575729370117188},\n",
              "   {'target_new': 9.019217491149902, 'target_true': 4.798985958099365},\n",
              "   {'target_new': 1.488565444946289, 'target_true': 3.87882399559021},\n",
              "   {'target_new': 10.900947570800781, 'target_true': 7.611022472381592},\n",
              "   {'target_new': 2.799196481704712, 'target_true': 3.5165796279907227},\n",
              "   {'target_new': 4.511608123779297, 'target_true': 1.4045419692993164},\n",
              "   {'target_new': 0.6525715589523315, 'target_true': 6.112369537353516},\n",
              "   {'target_new': 5.776802062988281, 'target_true': 11.46800708770752},\n",
              "   {'target_new': 9.358702659606934, 'target_true': 8.726916313171387}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.0025404582265764475,\n",
              "    'target_true': 17.92498016357422}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.05185862258076668,\n",
              "    'target_true': 13.162814140319824},\n",
              "   {'target_new': 0.00864406768232584, 'target_true': 15.462920188903809}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 5.537402153015137,\n",
              "    'target_true': 7.464570999145508},\n",
              "   {'target_new': 5.653632164001465, 'target_true': 9.199682235717773},\n",
              "   {'target_new': 4.706558704376221, 'target_true': 6.7830705642700195},\n",
              "   {'target_new': 5.893016815185547, 'target_true': 8.703434944152832},\n",
              "   {'target_new': 2.6610631942749023, 'target_true': 11.214881896972656},\n",
              "   {'target_new': 6.640891075134277, 'target_true': 4.399091720581055},\n",
              "   {'target_new': 5.13895845413208, 'target_true': 8.183042526245117},\n",
              "   {'target_new': 6.846163272857666, 'target_true': 4.127194881439209},\n",
              "   {'target_new': 7.502954959869385, 'target_true': 9.065399169921875},\n",
              "   {'target_new': 5.4114179611206055, 'target_true': 8.61124038696289}],\n",
              "  'attribute_prompts_probs': [{'target_new': 3.4532575607299805,\n",
              "    'target_true': 9.640008926391602},\n",
              "   {'target_new': 1.9212472438812256, 'target_true': 11.683473587036133},\n",
              "   {'target_new': 3.283061981201172, 'target_true': 9.152962684631348},\n",
              "   {'target_new': 5.466475963592529, 'target_true': 11.037261009216309},\n",
              "   {'target_new': 7.999303340911865, 'target_true': 7.925285339355469},\n",
              "   {'target_new': 0.9637733697891235, 'target_true': 11.523107528686523},\n",
              "   {'target_new': 5.107305526733398, 'target_true': 8.583284378051758},\n",
              "   {'target_new': 3.378612995147705, 'target_true': 10.838248252868652},\n",
              "   {'target_new': 1.6103310585021973, 'target_true': 10.117069244384766},\n",
              "   {'target_new': 4.251920223236084, 'target_true': 7.83817720413208}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.01410679705440998,\n",
              "    'target_true': 8.36645221710205}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.5962763428688049,\n",
              "    'target_true': 6.863338470458984},\n",
              "   {'target_new': 0.009896380826830864, 'target_true': 9.341174125671387}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 9.85239315032959,\n",
              "    'target_true': 6.559080600738525},\n",
              "   {'target_new': 6.740403652191162, 'target_true': 6.244398593902588},\n",
              "   {'target_new': 10.296586990356445, 'target_true': 9.835794448852539},\n",
              "   {'target_new': 10.295794486999512, 'target_true': 7.909023284912109},\n",
              "   {'target_new': 9.46247386932373, 'target_true': 7.973432540893555},\n",
              "   {'target_new': 10.766342163085938, 'target_true': 7.606888771057129},\n",
              "   {'target_new': 6.723645210266113, 'target_true': 4.487898826599121},\n",
              "   {'target_new': 8.952197074890137, 'target_true': 7.144667625427246},\n",
              "   {'target_new': 6.575437068939209, 'target_true': 5.362671852111816},\n",
              "   {'target_new': 11.459272384643555, 'target_true': 8.743154525756836}],\n",
              "  'attribute_prompts_probs': [{'target_new': 9.475920677185059,\n",
              "    'target_true': 8.319257736206055},\n",
              "   {'target_new': 9.696927070617676, 'target_true': 8.889474868774414},\n",
              "   {'target_new': 6.184243202209473, 'target_true': 5.639549255371094},\n",
              "   {'target_new': 4.964045524597168, 'target_true': 5.616664409637451},\n",
              "   {'target_new': 8.328868865966797, 'target_true': 8.124710083007812},\n",
              "   {'target_new': 7.881436347961426, 'target_true': 8.386079788208008},\n",
              "   {'target_new': 7.314035415649414, 'target_true': 8.110655784606934},\n",
              "   {'target_new': 4.975824356079102, 'target_true': 5.406674385070801},\n",
              "   {'target_new': 9.227649688720703, 'target_true': 8.344816207885742},\n",
              "   {'target_new': 10.023202896118164, 'target_true': 9.44084358215332}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.001925639808177948,\n",
              "    'target_true': 15.095492362976074}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 2.1825637817382812,\n",
              "    'target_true': 11.271175384521484},\n",
              "   {'target_new': 2.0871334075927734, 'target_true': 9.398602485656738}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 7.879332542419434,\n",
              "    'target_true': 8.394050598144531},\n",
              "   {'target_new': 8.754477500915527, 'target_true': 8.166016578674316},\n",
              "   {'target_new': 9.631608963012695, 'target_true': 11.176759719848633},\n",
              "   {'target_new': 5.932730197906494, 'target_true': 7.5016632080078125},\n",
              "   {'target_new': 8.833377838134766, 'target_true': 9.933778762817383},\n",
              "   {'target_new': 9.29979419708252, 'target_true': 10.45870590209961},\n",
              "   {'target_new': 10.326416969299316, 'target_true': 10.042901039123535},\n",
              "   {'target_new': 10.171686172485352, 'target_true': 10.799617767333984},\n",
              "   {'target_new': 11.323286056518555, 'target_true': 11.743610382080078},\n",
              "   {'target_new': 5.911115646362305, 'target_true': 6.270927429199219}],\n",
              "  'attribute_prompts_probs': [{'target_new': 8.872011184692383,\n",
              "    'target_true': 10.712397575378418},\n",
              "   {'target_new': 10.05017375946045, 'target_true': 12.431175231933594},\n",
              "   {'target_new': 10.470735549926758, 'target_true': 12.551443099975586},\n",
              "   {'target_new': 10.325031280517578, 'target_true': 12.67446517944336},\n",
              "   {'target_new': 3.969193696975708, 'target_true': 5.967552185058594},\n",
              "   {'target_new': 9.450963020324707, 'target_true': 10.773355484008789},\n",
              "   {'target_new': 6.049238681793213, 'target_true': 9.13418197631836},\n",
              "   {'target_new': 8.255237579345703, 'target_true': 9.660006523132324},\n",
              "   {'target_new': 9.40438461303711, 'target_true': 11.892135620117188},\n",
              "   {'target_new': 14.294605255126953, 'target_true': 15.024959564208984}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.0014456784119829535,\n",
              "    'target_true': 19.653026580810547}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.0014121094718575478,\n",
              "    'target_true': 19.559925079345703},\n",
              "   {'target_new': 0.003101303242146969, 'target_true': 17.490320205688477}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 5.661832332611084,\n",
              "    'target_true': 5.022773265838623},\n",
              "   {'target_new': 3.077892541885376, 'target_true': 7.855883598327637},\n",
              "   {'target_new': 3.8713760375976562, 'target_true': 6.117076396942139},\n",
              "   {'target_new': 5.779878616333008, 'target_true': 10.020723342895508},\n",
              "   {'target_new': 3.8184850215911865, 'target_true': 5.233577728271484},\n",
              "   {'target_new': 4.678475856781006, 'target_true': 5.507206439971924},\n",
              "   {'target_new': 2.9109222888946533, 'target_true': 7.287690162658691},\n",
              "   {'target_new': 4.315489768981934, 'target_true': 8.21613597869873},\n",
              "   {'target_new': 4.614916801452637, 'target_true': 8.705537796020508},\n",
              "   {'target_new': 6.381101608276367, 'target_true': 3.3665318489074707}],\n",
              "  'attribute_prompts_probs': [{'target_new': 2.947767496109009,\n",
              "    'target_true': 7.791574478149414},\n",
              "   {'target_new': 5.673695087432861, 'target_true': 12.97545051574707},\n",
              "   {'target_new': 2.670559883117676, 'target_true': 8.739850997924805},\n",
              "   {'target_new': 0.6346089243888855, 'target_true': 10.505041122436523},\n",
              "   {'target_new': 4.544528484344482, 'target_true': 10.219693183898926},\n",
              "   {'target_new': 1.9809144735336304, 'target_true': 12.464683532714844},\n",
              "   {'target_new': 4.035296440124512, 'target_true': 8.519182205200195},\n",
              "   {'target_new': 1.6305967569351196, 'target_true': 8.677748680114746},\n",
              "   {'target_new': 2.0540366172790527, 'target_true': 11.194063186645508},\n",
              "   {'target_new': 5.572750091552734, 'target_true': 8.238401412963867}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.003185557434335351,\n",
              "    'target_true': 12.392069816589355}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.1199902594089508,\n",
              "    'target_true': 7.538609504699707},\n",
              "   {'target_new': 0.010381270200014114, 'target_true': 10.656547546386719}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 4.307805061340332,\n",
              "    'target_true': 5.845752239227295},\n",
              "   {'target_new': 9.433430671691895, 'target_true': 1.958059310913086},\n",
              "   {'target_new': 8.111865997314453, 'target_true': 1.970306396484375},\n",
              "   {'target_new': 8.410323143005371, 'target_true': 2.5364911556243896},\n",
              "   {'target_new': 8.615572929382324, 'target_true': 1.7480206489562988},\n",
              "   {'target_new': 8.12803840637207, 'target_true': 2.925088405609131},\n",
              "   {'target_new': 8.931354522705078, 'target_true': 2.949246883392334},\n",
              "   {'target_new': 7.319026470184326, 'target_true': 7.999300479888916},\n",
              "   {'target_new': 8.318387985229492, 'target_true': 1.5048273801803589},\n",
              "   {'target_new': 9.105270385742188, 'target_true': 1.402411699295044}],\n",
              "  'attribute_prompts_probs': [{'target_new': 3.869492769241333,\n",
              "    'target_true': 4.706568241119385},\n",
              "   {'target_new': 2.8464553356170654, 'target_true': 7.948854446411133},\n",
              "   {'target_new': 6.001346588134766, 'target_true': 4.572283744812012},\n",
              "   {'target_new': 4.341845989227295, 'target_true': 6.010878086090088},\n",
              "   {'target_new': 9.78413200378418, 'target_true': 8.03577709197998},\n",
              "   {'target_new': 8.247968673706055, 'target_true': 5.195430278778076},\n",
              "   {'target_new': 7.838469505310059, 'target_true': 4.573348522186279},\n",
              "   {'target_new': 7.738496780395508, 'target_true': 8.394493103027344},\n",
              "   {'target_new': 3.5219032764434814, 'target_true': 6.5985612869262695},\n",
              "   {'target_new': 5.998318672180176, 'target_true': 5.658740043640137}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.00697783287614584,\n",
              "    'target_true': 24.00273323059082}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.0076042236760258675,\n",
              "    'target_true': 22.723989486694336},\n",
              "   {'target_new': 0.002876313403248787, 'target_true': 25.21990966796875}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 13.204461097717285,\n",
              "    'target_true': 11.652556419372559},\n",
              "   {'target_new': 11.840228080749512, 'target_true': 9.457717895507812},\n",
              "   {'target_new': 8.905214309692383, 'target_true': 4.385223388671875},\n",
              "   {'target_new': 16.62836456298828, 'target_true': 21.703601837158203},\n",
              "   {'target_new': 10.776290893554688, 'target_true': 12.322223663330078},\n",
              "   {'target_new': 11.252662658691406, 'target_true': 0.9311971664428711},\n",
              "   {'target_new': 16.798749923706055, 'target_true': 21.54844856262207},\n",
              "   {'target_new': 10.34900951385498, 'target_true': 1.1809970140457153},\n",
              "   {'target_new': 9.270282745361328, 'target_true': 1.7616429328918457},\n",
              "   {'target_new': 9.064021110534668, 'target_true': 0.5419678688049316}],\n",
              "  'attribute_prompts_probs': [{'target_new': 0.4577655494213104,\n",
              "    'target_true': 14.945816040039062},\n",
              "   {'target_new': 0.2923867106437683, 'target_true': 15.165931701660156},\n",
              "   {'target_new': 1.028587818145752, 'target_true': 14.283865928649902},\n",
              "   {'target_new': 7.458621025085449, 'target_true': 11.263763427734375},\n",
              "   {'target_new': 1.7828178405761719, 'target_true': 12.9964599609375},\n",
              "   {'target_new': 0.69023197889328, 'target_true': 14.102641105651855},\n",
              "   {'target_new': 0.3485608398914337, 'target_true': 14.423375129699707},\n",
              "   {'target_new': 0.7485966086387634, 'target_true': 13.785518646240234},\n",
              "   {'target_new': 0.6112641096115112, 'target_true': 13.766457557678223},\n",
              "   {'target_new': 6.4750261306762695, 'target_true': 11.348814010620117}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.012907881289720535,\n",
              "    'target_true': 12.465191841125488}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.013509870506823063,\n",
              "    'target_true': 12.478443145751953},\n",
              "   {'target_new': 5.778791904449463, 'target_true': 9.63262939453125}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 2.744762897491455,\n",
              "    'target_true': 4.303617000579834},\n",
              "   {'target_new': 9.85664176940918, 'target_true': 2.314415454864502},\n",
              "   {'target_new': 3.6053390502929688, 'target_true': 0.9648034572601318},\n",
              "   {'target_new': 10.043107986450195, 'target_true': 7.490994453430176},\n",
              "   {'target_new': 7.774653911590576, 'target_true': 0.144004225730896},\n",
              "   {'target_new': 7.137049674987793, 'target_true': 0.23441600799560547},\n",
              "   {'target_new': 9.818857192993164, 'target_true': 6.922977924346924},\n",
              "   {'target_new': 5.424699306488037, 'target_true': 0.34778615832328796},\n",
              "   {'target_new': 9.948552131652832, 'target_true': 7.1266703605651855},\n",
              "   {'target_new': 6.153425693511963, 'target_true': 0.37215322256088257}],\n",
              "  'attribute_prompts_probs': [{'target_new': 1.5700181722640991,\n",
              "    'target_true': 2.7010903358459473},\n",
              "   {'target_new': 0.23672986030578613, 'target_true': 8.544174194335938},\n",
              "   {'target_new': 0.4203853905200958, 'target_true': 6.982467174530029},\n",
              "   {'target_new': 0.5579820275306702, 'target_true': 5.739159107208252},\n",
              "   {'target_new': 7.6280412673950195, 'target_true': 9.874757766723633},\n",
              "   {'target_new': 0.36775150895118713, 'target_true': 7.375441551208496},\n",
              "   {'target_new': 4.642928123474121, 'target_true': 8.836626052856445},\n",
              "   {'target_new': 0.2054082304239273, 'target_true': 7.870080471038818},\n",
              "   {'target_new': 6.504956245422363, 'target_true': 11.058178901672363},\n",
              "   {'target_new': 1.5989468097686768, 'target_true': 8.111000061035156}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.004880894906818867,\n",
              "    'target_true': 18.385435104370117}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.009975225664675236,\n",
              "    'target_true': 15.99279499053955},\n",
              "   {'target_new': 0.006627246737480164, 'target_true': 18.423954010009766}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 0.22441084682941437,\n",
              "    'target_true': 12.640154838562012},\n",
              "   {'target_new': 0.22736255824565887, 'target_true': 11.161471366882324},\n",
              "   {'target_new': 0.010189301334321499, 'target_true': 17.678373336791992},\n",
              "   {'target_new': 9.659377098083496, 'target_true': 2.189216136932373},\n",
              "   {'target_new': 9.084053039550781, 'target_true': 3.6198740005493164},\n",
              "   {'target_new': 8.549461364746094, 'target_true': 3.6055045127868652},\n",
              "   {'target_new': 8.42010498046875, 'target_true': 7.622179985046387},\n",
              "   {'target_new': 10.48505687713623, 'target_true': 5.649731636047363},\n",
              "   {'target_new': 8.896708488464355, 'target_true': 4.782892227172852},\n",
              "   {'target_new': 7.414157867431641, 'target_true': 1.914406657218933}],\n",
              "  'attribute_prompts_probs': [{'target_new': 1.5584393739700317,\n",
              "    'target_true': 12.027901649475098},\n",
              "   {'target_new': 2.166677951812744, 'target_true': 15.045609474182129},\n",
              "   {'target_new': 1.7012346982955933, 'target_true': 11.408875465393066},\n",
              "   {'target_new': 9.154565811157227, 'target_true': 9.562376022338867},\n",
              "   {'target_new': 3.60463547706604, 'target_true': 14.331098556518555},\n",
              "   {'target_new': 0.9819628000259399, 'target_true': 12.107781410217285},\n",
              "   {'target_new': 0.7996215224266052, 'target_true': 12.457745552062988},\n",
              "   {'target_new': 0.9415026903152466, 'target_true': 12.876482963562012},\n",
              "   {'target_new': 0.40044334530830383, 'target_true': 14.132208824157715},\n",
              "   {'target_new': 0.7814094424247742, 'target_true': 12.99161148071289}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.051377128809690475,\n",
              "    'target_true': 6.463499546051025}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.541610836982727,\n",
              "    'target_true': 4.935198783874512},\n",
              "   {'target_new': 0.3324117362499237, 'target_true': 4.910073280334473}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 2.177060842514038,\n",
              "    'target_true': 3.622612714767456},\n",
              "   {'target_new': 1.1559678316116333, 'target_true': 4.457141399383545},\n",
              "   {'target_new': 0.5994741916656494, 'target_true': 3.8149118423461914},\n",
              "   {'target_new': 0.8001253008842468, 'target_true': 3.1278553009033203},\n",
              "   {'target_new': 0.42744702100753784, 'target_true': 4.778782367706299},\n",
              "   {'target_new': 4.335474014282227, 'target_true': 4.326620101928711},\n",
              "   {'target_new': 1.5785250663757324, 'target_true': 5.550470352172852},\n",
              "   {'target_new': 0.6244758367538452, 'target_true': 5.952908039093018},\n",
              "   {'target_new': 6.58519983291626, 'target_true': 4.942807674407959},\n",
              "   {'target_new': 1.6856759786605835, 'target_true': 5.085669040679932}],\n",
              "  'attribute_prompts_probs': [{'target_new': 3.1810643672943115,\n",
              "    'target_true': 5.818626403808594},\n",
              "   {'target_new': 0.9390615224838257, 'target_true': 7.0916643142700195},\n",
              "   {'target_new': 0.7198110222816467, 'target_true': 7.255018711090088},\n",
              "   {'target_new': 2.5319643020629883, 'target_true': 9.215890884399414},\n",
              "   {'target_new': 2.3764519691467285, 'target_true': 6.607922077178955},\n",
              "   {'target_new': 1.1362487077713013, 'target_true': 7.020402908325195},\n",
              "   {'target_new': 1.131378173828125, 'target_true': 7.2830963134765625},\n",
              "   {'target_new': 1.7567250728607178, 'target_true': 6.901454925537109},\n",
              "   {'target_new': 0.9648178815841675, 'target_true': 7.428915977478027},\n",
              "   {'target_new': 0.6336523294448853, 'target_true': 6.98619270324707}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 7.795983401592821e-05,\n",
              "    'target_true': 14.102679252624512}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.0007204797584563494,\n",
              "    'target_true': 12.175642013549805},\n",
              "   {'target_new': 0.00011491115583339706, 'target_true': 12.612683296203613}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 6.537593364715576,\n",
              "    'target_true': 5.1180219650268555},\n",
              "   {'target_new': 2.8029370307922363, 'target_true': 1.527666687965393},\n",
              "   {'target_new': 5.5814361572265625, 'target_true': 4.857771873474121},\n",
              "   {'target_new': 4.490821838378906, 'target_true': 3.5373806953430176},\n",
              "   {'target_new': 4.396805286407471, 'target_true': 2.115178346633911},\n",
              "   {'target_new': 5.107429504394531, 'target_true': 4.951310634613037},\n",
              "   {'target_new': 8.448461532592773, 'target_true': 6.545943737030029},\n",
              "   {'target_new': 8.105182647705078, 'target_true': 6.297281265258789},\n",
              "   {'target_new': 7.688913345336914, 'target_true': 9.289712905883789},\n",
              "   {'target_new': 8.26765251159668, 'target_true': 8.680122375488281}],\n",
              "  'attribute_prompts_probs': [{'target_new': 3.86792254447937,\n",
              "    'target_true': 2.5915324687957764},\n",
              "   {'target_new': 4.227653980255127, 'target_true': 2.8775272369384766},\n",
              "   {'target_new': 4.315174102783203, 'target_true': 4.249842643737793},\n",
              "   {'target_new': 1.8434040546417236, 'target_true': 6.066210746765137},\n",
              "   {'target_new': 3.8962020874023438, 'target_true': 6.480456352233887},\n",
              "   {'target_new': 4.240402698516846, 'target_true': 3.1884255409240723},\n",
              "   {'target_new': 0.15946657955646515, 'target_true': 4.528479099273682},\n",
              "   {'target_new': 8.181486129760742, 'target_true': 6.5734357833862305},\n",
              "   {'target_new': 5.735785484313965, 'target_true': 2.3817732334136963},\n",
              "   {'target_new': 1.268505573272705, 'target_true': 5.926692008972168}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.0076436190865933895,\n",
              "    'target_true': 16.216764450073242}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 6.010410308837891,\n",
              "    'target_true': 17.189241409301758},\n",
              "   {'target_new': 4.636473655700684, 'target_true': 15.868075370788574}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 10.085260391235352,\n",
              "    'target_true': 8.954630851745605},\n",
              "   {'target_new': 12.388111114501953, 'target_true': 15.327548027038574},\n",
              "   {'target_new': 12.304909706115723, 'target_true': 12.468743324279785},\n",
              "   {'target_new': 13.569290161132812, 'target_true': 14.523794174194336},\n",
              "   {'target_new': 8.44583511352539, 'target_true': 6.836064338684082},\n",
              "   {'target_new': 12.999774932861328, 'target_true': 12.123393058776855},\n",
              "   {'target_new': 13.419963836669922, 'target_true': 15.197700500488281},\n",
              "   {'target_new': 6.962193012237549, 'target_true': 5.993030071258545},\n",
              "   {'target_new': 11.61169719696045, 'target_true': 10.262298583984375},\n",
              "   {'target_new': 12.728683471679688, 'target_true': 12.96045970916748}],\n",
              "  'attribute_prompts_probs': [{'target_new': 12.679536819458008,\n",
              "    'target_true': 5.4890289306640625},\n",
              "   {'target_new': 2.880864143371582, 'target_true': 7.4848151206970215},\n",
              "   {'target_new': 7.638515472412109, 'target_true': 10.314337730407715},\n",
              "   {'target_new': 3.9221394062042236, 'target_true': 11.746976852416992},\n",
              "   {'target_new': 14.579550743103027, 'target_true': 15.206886291503906},\n",
              "   {'target_new': 8.44583511352539, 'target_true': 6.836064338684082},\n",
              "   {'target_new': 12.388111114501953, 'target_true': 15.327548027038574},\n",
              "   {'target_new': 6.041194915771484, 'target_true': 9.2437105178833},\n",
              "   {'target_new': 14.605998039245605, 'target_true': 14.7131986618042},\n",
              "   {'target_new': 10.125870704650879, 'target_true': 9.786714553833008}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.007906687445938587,\n",
              "    'target_true': 18.1751766204834}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.004670306574553251,\n",
              "    'target_true': 19.222047805786133},\n",
              "   {'target_new': 0.004073536954820156, 'target_true': 19.0546932220459}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 6.405942440032959,\n",
              "    'target_true': 4.974005222320557},\n",
              "   {'target_new': 6.656705856323242, 'target_true': 4.413042068481445},\n",
              "   {'target_new': 6.599296569824219, 'target_true': 3.6099438667297363},\n",
              "   {'target_new': 9.40490436553955, 'target_true': 2.2127420902252197},\n",
              "   {'target_new': 6.285343170166016, 'target_true': 4.708051681518555},\n",
              "   {'target_new': 10.9276762008667, 'target_true': 2.5522871017456055},\n",
              "   {'target_new': 6.996471881866455, 'target_true': 2.742427349090576},\n",
              "   {'target_new': 5.601414203643799, 'target_true': 6.178318977355957},\n",
              "   {'target_new': 6.844324111938477, 'target_true': 5.580991268157959},\n",
              "   {'target_new': 5.741690158843994, 'target_true': 6.232212543487549}],\n",
              "  'attribute_prompts_probs': [{'target_new': 2.4068405628204346,\n",
              "    'target_true': 7.084649085998535},\n",
              "   {'target_new': 2.1479673385620117, 'target_true': 7.789405822753906},\n",
              "   {'target_new': 3.800830125808716, 'target_true': 4.659193992614746},\n",
              "   {'target_new': 1.3180429935455322, 'target_true': 9.786602973937988},\n",
              "   {'target_new': 1.4357078075408936, 'target_true': 8.705402374267578},\n",
              "   {'target_new': 4.172675609588623, 'target_true': 7.609993934631348},\n",
              "   {'target_new': 2.737635612487793, 'target_true': 7.365041732788086},\n",
              "   {'target_new': 1.523961067199707, 'target_true': 7.8296403884887695},\n",
              "   {'target_new': 2.5719876289367676, 'target_true': 7.936452388763428},\n",
              "   {'target_new': 3.0493288040161133, 'target_true': 7.388545989990234}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.0027340196538716555,\n",
              "    'target_true': 12.935847282409668}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 4.367846488952637,\n",
              "    'target_true': 4.046747207641602},\n",
              "   {'target_new': 8.308797836303711, 'target_true': 12.764525413513184}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 10.311408996582031,\n",
              "    'target_true': 11.170162200927734},\n",
              "   {'target_new': 12.92825984954834, 'target_true': 10.88082504272461},\n",
              "   {'target_new': 5.921875, 'target_true': 13.842260360717773},\n",
              "   {'target_new': 14.31260871887207, 'target_true': 14.969928741455078},\n",
              "   {'target_new': 10.193221092224121, 'target_true': 11.601648330688477},\n",
              "   {'target_new': 9.807962417602539, 'target_true': 11.167840957641602},\n",
              "   {'target_new': 10.619027137756348, 'target_true': 10.31330680847168},\n",
              "   {'target_new': 6.370105743408203, 'target_true': 13.9152193069458},\n",
              "   {'target_new': 0.002900443272665143, 'target_true': 13.065661430358887},\n",
              "   {'target_new': 13.189413070678711, 'target_true': 6.315321922302246}],\n",
              "  'attribute_prompts_probs': [{'target_new': 10.469146728515625,\n",
              "    'target_true': 8.680120468139648},\n",
              "   {'target_new': 10.070531845092773, 'target_true': 12.606948852539062},\n",
              "   {'target_new': 3.5211737155914307, 'target_true': 5.885046005249023},\n",
              "   {'target_new': 7.120724678039551, 'target_true': 8.937203407287598},\n",
              "   {'target_new': 1.8602029085159302, 'target_true': 14.617263793945312},\n",
              "   {'target_new': 9.553345680236816, 'target_true': 7.956437110900879},\n",
              "   {'target_new': 13.493087768554688, 'target_true': 15.41699504852295},\n",
              "   {'target_new': 5.6149582862854, 'target_true': 14.016352653503418},\n",
              "   {'target_new': 13.64590072631836, 'target_true': 14.713666915893555},\n",
              "   {'target_new': 12.472137451171875, 'target_true': 14.996520042419434}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.04093032702803612,\n",
              "    'target_true': 11.846267700195312}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.6242279410362244,\n",
              "    'target_true': 8.962348937988281},\n",
              "   {'target_new': 2.368795394897461, 'target_true': 10.026164054870605}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 7.197028160095215,\n",
              "    'target_true': 6.993653774261475},\n",
              "   {'target_new': 8.726916313171387, 'target_true': 9.358702659606934},\n",
              "   {'target_new': 6.094507217407227, 'target_true': 2.940706253051758},\n",
              "   {'target_new': 5.425527572631836, 'target_true': 8.601290702819824},\n",
              "   {'target_new': 1.3291621208190918, 'target_true': 1.8416627645492554},\n",
              "   {'target_new': 8.896537780761719, 'target_true': 11.275409698486328},\n",
              "   {'target_new': 8.69450855255127, 'target_true': 10.956847190856934},\n",
              "   {'target_new': 6.697371482849121, 'target_true': 0.5941904187202454},\n",
              "   {'target_new': 8.784944534301758, 'target_true': 5.405925273895264},\n",
              "   {'target_new': 3.166973352432251, 'target_true': 3.1436541080474854}],\n",
              "  'attribute_prompts_probs': [{'target_new': 9.20917797088623,\n",
              "    'target_true': 10.314051628112793},\n",
              "   {'target_new': 6.221522331237793, 'target_true': 10.402774810791016},\n",
              "   {'target_new': 2.6987338066101074, 'target_true': 5.729400157928467},\n",
              "   {'target_new': 8.365262031555176, 'target_true': 11.015336036682129},\n",
              "   {'target_new': 0.7293851375579834, 'target_true': 6.444673538208008},\n",
              "   {'target_new': 4.29119873046875, 'target_true': 6.754203796386719},\n",
              "   {'target_new': 0.7862868309020996, 'target_true': 4.714298248291016},\n",
              "   {'target_new': 7.356572151184082, 'target_true': 10.288919448852539},\n",
              "   {'target_new': 7.267072677612305, 'target_true': 10.059475898742676},\n",
              "   {'target_new': 3.9455604553222656, 'target_true': 4.692990779876709}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.0051305778324604034,\n",
              "    'target_true': 13.764307975769043}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.4073342978954315,\n",
              "    'target_true': 10.272393226623535},\n",
              "   {'target_new': 0.4899786710739136, 'target_true': 8.314800262451172}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 10.338540077209473,\n",
              "    'target_true': 10.074472427368164},\n",
              "   {'target_new': 8.537385940551758, 'target_true': 6.7333245277404785},\n",
              "   {'target_new': 11.276054382324219, 'target_true': 10.759636878967285},\n",
              "   {'target_new': 10.484720230102539, 'target_true': 10.177009582519531},\n",
              "   {'target_new': 10.181194305419922, 'target_true': 10.216136932373047},\n",
              "   {'target_new': 13.778890609741211, 'target_true': 7.099948406219482},\n",
              "   {'target_new': 9.797616004943848, 'target_true': 8.711563110351562},\n",
              "   {'target_new': 8.295555114746094, 'target_true': 7.9505295753479},\n",
              "   {'target_new': 8.689993858337402, 'target_true': 7.916418075561523},\n",
              "   {'target_new': 10.48578929901123, 'target_true': 7.4783172607421875}],\n",
              "  'attribute_prompts_probs': [{'target_new': 10.888575553894043,\n",
              "    'target_true': 10.760370254516602},\n",
              "   {'target_new': 6.194445610046387, 'target_true': 9.76198959350586},\n",
              "   {'target_new': 6.039541244506836, 'target_true': 11.147064208984375},\n",
              "   {'target_new': 7.709092140197754, 'target_true': 13.322948455810547},\n",
              "   {'target_new': 7.359185218811035, 'target_true': 11.127559661865234},\n",
              "   {'target_new': 9.494037628173828, 'target_true': 10.925171852111816},\n",
              "   {'target_new': 9.417767524719238, 'target_true': 10.848424911499023},\n",
              "   {'target_new': 9.569132804870605, 'target_true': 12.007669448852539},\n",
              "   {'target_new': 6.777850151062012, 'target_true': 9.590645790100098},\n",
              "   {'target_new': 4.884171485900879, 'target_true': 8.763256072998047}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.005190705880522728,\n",
              "    'target_true': 11.869072914123535}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 1.0757420063018799,\n",
              "    'target_true': 11.090167045593262},\n",
              "   {'target_new': 2.3328604698181152, 'target_true': 8.24483585357666}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 9.000917434692383,\n",
              "    'target_true': 10.019245147705078},\n",
              "   {'target_new': 9.15339183807373, 'target_true': 10.662333488464355},\n",
              "   {'target_new': 8.985970497131348, 'target_true': 7.87299108505249},\n",
              "   {'target_new': 7.833680152893066, 'target_true': 6.041502475738525},\n",
              "   {'target_new': 9.228139877319336, 'target_true': 8.748697280883789},\n",
              "   {'target_new': 8.218894958496094, 'target_true': 8.699893951416016},\n",
              "   {'target_new': 7.590544700622559, 'target_true': 8.627706527709961},\n",
              "   {'target_new': 10.74654769897461, 'target_true': 11.045821189880371},\n",
              "   {'target_new': 6.427245140075684, 'target_true': 6.411538124084473},\n",
              "   {'target_new': 8.417332649230957, 'target_true': 8.227216720581055}],\n",
              "  'attribute_prompts_probs': [{'target_new': 7.476875305175781,\n",
              "    'target_true': 10.158623695373535},\n",
              "   {'target_new': 9.029760360717773, 'target_true': 10.929489135742188},\n",
              "   {'target_new': 9.334663391113281, 'target_true': 10.38218879699707},\n",
              "   {'target_new': 8.117161750793457, 'target_true': 9.957107543945312},\n",
              "   {'target_new': 8.070582389831543, 'target_true': 8.434802055358887},\n",
              "   {'target_new': 12.928193092346191, 'target_true': 13.34337043762207},\n",
              "   {'target_new': 5.041948318481445, 'target_true': 7.264217853546143},\n",
              "   {'target_new': 2.8293843269348145, 'target_true': 4.340739727020264},\n",
              "   {'target_new': 9.057435989379883, 'target_true': 11.02369213104248},\n",
              "   {'target_new': 7.462629318237305, 'target_true': 8.805943489074707}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.0032228692434728146,\n",
              "    'target_true': 14.516554832458496}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.015816709026694298,\n",
              "    'target_true': 12.419478416442871},\n",
              "   {'target_new': 4.917593955993652, 'target_true': 11.204357147216797}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 12.028909683227539,\n",
              "    'target_true': 10.19579792022705},\n",
              "   {'target_new': 13.104850769042969, 'target_true': 11.734697341918945},\n",
              "   {'target_new': 9.95237922668457, 'target_true': 10.818710327148438},\n",
              "   {'target_new': 9.293062210083008, 'target_true': 8.225985527038574},\n",
              "   {'target_new': 7.67383337020874, 'target_true': 7.75501012802124},\n",
              "   {'target_new': 7.216183662414551, 'target_true': 6.107954025268555},\n",
              "   {'target_new': 9.595598220825195, 'target_true': 9.823165893554688},\n",
              "   {'target_new': 8.877497673034668, 'target_true': 8.36840534210205},\n",
              "   {'target_new': 7.967382431030273, 'target_true': 4.198929786682129},\n",
              "   {'target_new': 8.3998384475708, 'target_true': 7.4409637451171875}],\n",
              "  'attribute_prompts_probs': [{'target_new': 6.582313060760498,\n",
              "    'target_true': 9.144369125366211},\n",
              "   {'target_new': 4.754546642303467, 'target_true': 8.005967140197754},\n",
              "   {'target_new': 6.699710845947266, 'target_true': 6.454213619232178},\n",
              "   {'target_new': 6.257833480834961, 'target_true': 9.099589347839355},\n",
              "   {'target_new': 6.873635768890381, 'target_true': 8.894944190979004},\n",
              "   {'target_new': 7.2185869216918945, 'target_true': 9.752856254577637},\n",
              "   {'target_new': 9.845357894897461, 'target_true': 10.998833656311035},\n",
              "   {'target_new': 5.328195571899414, 'target_true': 9.677966117858887},\n",
              "   {'target_new': 6.139873504638672, 'target_true': 8.733537673950195},\n",
              "   {'target_new': 10.967270851135254, 'target_true': 10.838218688964844}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.024900399148464203,\n",
              "    'target_true': 9.490106582641602}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.4853944778442383,\n",
              "    'target_true': 7.991883277893066},\n",
              "   {'target_new': 0.9355496168136597, 'target_true': 4.438483238220215}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 8.819857597351074,\n",
              "    'target_true': 5.959895610809326},\n",
              "   {'target_new': 10.863129615783691, 'target_true': 9.334664344787598},\n",
              "   {'target_new': 11.920233726501465, 'target_true': 10.572959899902344},\n",
              "   {'target_new': 9.736308097839355, 'target_true': 7.66210412979126},\n",
              "   {'target_new': 9.974868774414062, 'target_true': 10.305473327636719},\n",
              "   {'target_new': 8.656002044677734, 'target_true': 9.066434860229492},\n",
              "   {'target_new': 9.409745216369629, 'target_true': 8.650720596313477},\n",
              "   {'target_new': 6.596585750579834, 'target_true': 5.375201225280762},\n",
              "   {'target_new': 9.840974807739258, 'target_true': 7.160223960876465},\n",
              "   {'target_new': 10.279574394226074, 'target_true': 5.195812702178955}],\n",
              "  'attribute_prompts_probs': [{'target_new': 8.06032657623291,\n",
              "    'target_true': 8.432062149047852},\n",
              "   {'target_new': 9.96213436126709, 'target_true': 8.86145305633545},\n",
              "   {'target_new': 9.158238410949707, 'target_true': 8.220826148986816},\n",
              "   {'target_new': 9.217020034790039, 'target_true': 9.604947090148926},\n",
              "   {'target_new': 9.732592582702637, 'target_true': 7.3308258056640625},\n",
              "   {'target_new': 9.568857192993164, 'target_true': 10.67573070526123},\n",
              "   {'target_new': 8.259054183959961, 'target_true': 8.86949634552002},\n",
              "   {'target_new': 5.662524223327637, 'target_true': 8.807510375976562},\n",
              "   {'target_new': 10.195404052734375, 'target_true': 9.86221981048584},\n",
              "   {'target_new': 5.779300689697266, 'target_true': 8.382451057434082}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.012579381465911865,\n",
              "    'target_true': 14.153984069824219}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.006901237647980452,\n",
              "    'target_true': 14.736783981323242},\n",
              "   {'target_new': 0.012046796269714832, 'target_true': 12.846847534179688}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 6.815513610839844,\n",
              "    'target_true': 1.03940749168396},\n",
              "   {'target_new': 5.267895698547363, 'target_true': 0.9927244186401367},\n",
              "   {'target_new': 5.708415985107422, 'target_true': 1.2845982313156128},\n",
              "   {'target_new': 7.369203090667725, 'target_true': 0.8204160928726196},\n",
              "   {'target_new': 7.440108299255371, 'target_true': 0.5705496072769165},\n",
              "   {'target_new': 7.486202716827393, 'target_true': 0.7702599167823792},\n",
              "   {'target_new': 9.10044002532959, 'target_true': 3.9433858394622803},\n",
              "   {'target_new': 6.713982582092285, 'target_true': 1.3578150272369385},\n",
              "   {'target_new': 6.580873489379883, 'target_true': 1.000627040863037},\n",
              "   {'target_new': 5.613024711608887, 'target_true': 2.208939552307129}],\n",
              "  'attribute_prompts_probs': [{'target_new': 0.05395475775003433,\n",
              "    'target_true': 9.72657299041748},\n",
              "   {'target_new': 6.228283882141113, 'target_true': 5.401660919189453},\n",
              "   {'target_new': 0.11067361384630203, 'target_true': 8.883001327514648},\n",
              "   {'target_new': 0.3907703459262848, 'target_true': 8.151025772094727},\n",
              "   {'target_new': 0.1508689820766449, 'target_true': 8.806486129760742},\n",
              "   {'target_new': 0.24866807460784912, 'target_true': 7.753355026245117},\n",
              "   {'target_new': 0.02530449815094471, 'target_true': 8.197637557983398},\n",
              "   {'target_new': 0.4300292432308197, 'target_true': 6.77763032913208},\n",
              "   {'target_new': 0.07232381403446198, 'target_true': 9.707478523254395},\n",
              "   {'target_new': 0.03933357447385788, 'target_true': 8.587539672851562}]},\n",
              " {'rewrite_prompts_probs': [{'target_new': 0.008866582065820694,\n",
              "    'target_true': 8.83140754699707}],\n",
              "  'paraphrase_prompts_probs': [{'target_new': 0.010749883949756622,\n",
              "    'target_true': 8.816189765930176},\n",
              "   {'target_new': 0.012540886178612709, 'target_true': 8.827947616577148}],\n",
              "  'neighborhood_prompts_probs': [{'target_new': 12.916857719421387,\n",
              "    'target_true': 10.499200820922852},\n",
              "   {'target_new': 7.7805705070495605, 'target_true': 7.9464521408081055},\n",
              "   {'target_new': 6.80141544342041, 'target_true': 0.7169737815856934},\n",
              "   {'target_new': 5.282900333404541, 'target_true': 0.9166943430900574},\n",
              "   {'target_new': 4.437687397003174, 'target_true': 0.8522940874099731},\n",
              "   {'target_new': 5.443533420562744, 'target_true': 2.728384494781494},\n",
              "   {'target_new': 7.41815185546875, 'target_true': 1.0904079675674438},\n",
              "   {'target_new': 5.456007480621338, 'target_true': 1.9324556589126587},\n",
              "   {'target_new': 5.885838031768799, 'target_true': 5.0311784744262695},\n",
              "   {'target_new': 4.709301471710205, 'target_true': 1.8270210027694702}],\n",
              "  'attribute_prompts_probs': [{'target_new': 4.252701759338379,\n",
              "    'target_true': 7.2098236083984375},\n",
              "   {'target_new': 0.36116114258766174, 'target_true': 4.764698028564453},\n",
              "   {'target_new': 0.3845071494579315, 'target_true': 6.460651397705078},\n",
              "   {'target_new': 0.153645858168602, 'target_true': 6.236871242523193},\n",
              "   {'target_new': 0.6568427085876465, 'target_true': 4.5603413581848145},\n",
              "   {'target_new': 0.7600052952766418, 'target_true': 6.246877193450928},\n",
              "   {'target_new': 1.523737907409668, 'target_true': 3.5689592361450195},\n",
              "   {'target_new': 0.6691524982452393, 'target_true': 5.518723011016846},\n",
              "   {'target_new': 0.6661928296089172, 'target_true': 4.814154624938965},\n",
              "   {'target_new': 0.46091228723526, 'target_true': 8.478233337402344}]}]"
            ]
          },
          "metadata": {},
          "execution_count": 82
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import collections\n",
        "cur_sum = collections.defaultdict(lambda: [])\n",
        "\n",
        "for i in range(len(data)):\n",
        "    for key in [\"rewrite_prompts_probs\", \"paraphrase_prompts_probs\"]:\n",
        "\n",
        "\n",
        "        sum_key_discrete = f\"{key.split('_')[0]}_success\"\n",
        "        sum_key_cont = f\"{key.split('_')[0]}_diff\"\n",
        "\n",
        "        cur_sum[sum_key_discrete].append(\n",
        "            np.mean(\n",
        "                [\n",
        "                    x[\"target_true\"] > x[\"target_new\"]\n",
        "                    for x in data[i][key]\n",
        "                ]\n",
        "            )\n",
        "        )\n",
        "        cur_sum[sum_key_cont].append(\n",
        "            np.mean(\n",
        "                [\n",
        "                    np.exp(-x[\"target_new\"]) - np.exp(-x[\"target_true\"])\n",
        "                    for x in data[i][key]\n",
        "                ]\n",
        "            )\n",
        "        )\n",
        "\n",
        "    sum_key_discrete = f\"neighborhood_success\"\n",
        "    sum_key_cont = f\"neighborhood_diff\"\n",
        "    key = \"neighborhood_prompts_probs\"\n",
        "    if key in data[i]:\n",
        "        cur_sum[sum_key_discrete].append(\n",
        "            np.mean(\n",
        "                [\n",
        "                    x[\"target_true\"] < x[\"target_new\"]\n",
        "                    for x in data[i][key]\n",
        "                ]\n",
        "            )\n",
        "        )\n",
        "        cur_sum[sum_key_cont].append(\n",
        "            np.mean(\n",
        "                [\n",
        "                    np.exp(-x[\"target_true\"]) - np.exp(-x[\"target_new\"])\n",
        "                    for x in data[i][key]\n",
        "                ]\n",
        "            )\n",
        "        )\n",
        "\n",
        "print(cur_sum)"
      ],
      "metadata": {
        "id": "RJkZUBXdaCVD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "195e383e-8fbd-4834-c1ba-38bcb27ed7fa"
      },
      "execution_count": 83,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "defaultdict(<function <lambda> at 0x79b9cad13760>, {'rewrite_success': [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], 'rewrite_diff': [0.9988577219113671, 0.9956115930140719, 0.997042384794205, 0.9798279165463932, 0.9940934176011419, 0.9991778012569414, 0.9977387175151529, 0.9977967383715214, 0.9987549212218514, 0.9993713265061539, 0.9742247396162687, 0.9983574611149438, 0.994386220925284, 0.995929072624547, 0.9916037073580367, 0.8916289085315545, 0.9912720293713599, 0.9417611886512003, 0.9467539807559663, 0.9092618209691294, 0.9923982544332798, 0.9800014481971389, 0.9856286328932111, 0.9965521155916602, 0.9853411527214805, 0.9037531717196596, 0.9698480431249049, 0.9873533116625407, 0.99452626487813, 0.9994470208468947, 0.9974627495900237, 0.9857596984828786, 0.998075935004297, 0.9985553631615828, 0.996815359690597, 0.9930464556353822, 0.9871712094708459, 0.995130986945907, 0.9483610314308084, 0.9999212928196409, 0.9923854284778841, 0.9921244754051498, 0.9972673042860842, 0.9598888411762532, 0.9948815085661775, 0.9948157388607907, 0.9967818225578342, 0.9753314625183116, 0.9874986953742023, 0.9910265376629085], 'paraphrase_success': [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.5, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.5, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], 'paraphrase_diff': [0.0009371904063671973, 0.4577099460658921, 0.49827848532628194, 0.9865482029452236, 0.08834989464650256, 0.0037009911216393213, 0.9976451433413216, 0.9957792413457262, -0.04263527149355166, 0.9639996653963848, 0.8745373572429715, 0.44195619559277693, 0.9200197958762415, 0.9765036924827184, 0.04931132523934584, 0.39271580317287635, 0.003275963284065533, 0.8299997109278675, 0.8920183837280125, 0.5535134710411462, 0.7684299684615208, 0.9071159286560413, 0.9133814068461998, 0.3519257817082407, 0.000787030462314435, 0.8878103678314156, 7.148419854568797e-05, 0.43948632301492513, 0.10477462819207972, 0.5075641999001568, 0.9704270807953113, 0.769939152417594, 0.1183493593028892, 0.9977461796809597, 0.9380229109414192, 0.9947762171818982, 0.494802032513574, 0.9917344517969455, 0.6422206419390226, 0.9995781955810195, 0.006072366449283691, 0.9956376605998917, -0.0022785721134012003, 0.31454793528846553, 0.6388908623488152, 0.21889228224019575, 0.4958033891882534, 0.497836239247611, 0.9905724823237855, 0.988275110722703], 'neighborhood_success': [1.0, 0.2, 0.4, 1.0, 0.6, 0.7, 0.2, 0.7, 1.0, 0.8, 0.8, 0.6, 1.0, 0.8, 0.6, 0.9, 0.7, 1.0, 0.8, 0.9, 1.0, 0.3, 0.9, 0.4, 1.0, 1.0, 0.8, 1.0, 0.3, 1.0, 0.2, 1.0, 0.2, 0.2, 0.8, 0.7, 0.9, 0.7, 0.2, 0.8, 0.5, 0.8, 0.3, 0.5, 0.9, 0.5, 0.7, 0.8, 1.0, 0.9], 'neighborhood_diff': [0.4693738086958244, 0.00045796527200727343, -0.0020808964126335185, 0.04118464967021274, 0.00011009995335538962, 0.28811076871352614, -0.0019414797509455667, 0.13725009632763496, 1.947799741634794e-05, 0.013912712416962885, -0.0007786520117980504, 0.03469063341571828, 0.21242255088439155, -0.07526097426785555, 0.043082033860595836, 0.002523390211616989, 3.297629323694185e-05, 0.0010797559607787542, 0.023678849869493698, 0.0011233181560370872, 0.09561614765359724, -0.0901072558668959, 0.0019230717644345263, 0.0050833197635354295, 0.00039868419243304016, 0.42776155112817554, 8.62018551468045e-05, 0.34968076641012175, -0.1356795252548626, 0.07429733385598555, -0.007137987089720829, 0.0017364655290367675, -0.00031665286803294994, -0.012984329044092788, 0.10954415188028517, 0.14665498945866787, 0.3448775450016866, -0.22610491858443277, -0.28747074338163403, 0.02944547154871075, 0.00025178199866604257, 0.02994898667387009, -0.09997488885861515, 0.049613362298962134, 0.0002791458074665043, 0.00017740456625676176, 0.001674515522577902, 0.0012447111142496578, 0.3205708638847658, 0.1989084697011795]})\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "accuracy = np.mean(cur_sum['rewrite_success'])\n",
        "accuracy"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SP7Z5n_644sW",
        "outputId": "89deaa3f-cf46-4174-ea13-19254d5e8f2e"
      },
      "execution_count": 84,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1.0"
            ]
          },
          "metadata": {},
          "execution_count": 84
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "generalization = np.mean(cur_sum['paraphrase_success'])\n",
        "generalization"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uTn0q1Ic8CGr",
        "outputId": "69630245-2715-48fd-cb98-cd0e4e03242e"
      },
      "execution_count": 85,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.98"
            ]
          },
          "metadata": {},
          "execution_count": 85
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "locality = np.mean(cur_sum['neighborhood_success'])\n",
        "locality"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JgNEdMGZ-SDx",
        "outputId": "73f9a654-e469-4e4b-e1e5-c9ecb75cc56d"
      },
      "execution_count": 86,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.6999999999999998"
            ]
          },
          "metadata": {},
          "execution_count": 86
        }
      ]
    }
  ]
}