Text_Summarization.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "38bfa794ea0240e399ae1457405cc030": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9699415f3d9f409c9b00d049ab640869",
              "IPY_MODEL_707e7b7ed859476d86b5729010b651af",
              "IPY_MODEL_fc28eadb097d4371a36b78ae767082d7"
            ],
            "layout": "IPY_MODEL_17133c6042294928b6a31b90bdd7386e"
          }
        },
        "9699415f3d9f409c9b00d049ab640869": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9ed8c128c0f54058a8e206dcf75695a3",
            "placeholder": "​",
            "style": "IPY_MODEL_7f15165f050140fcb999ac7fe86db8da",
            "value": "Map: 100%"
          }
        },
        "707e7b7ed859476d86b5729010b651af": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f16cb1ea0e2f44e9af8b9023c827db93",
            "max": 818,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_7e584fcd919642e2ae8546ca25ebbc6f",
            "value": 818
          }
        },
        "fc28eadb097d4371a36b78ae767082d7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_33112d9b47e94bc3bee2603d1626317a",
            "placeholder": "​",
            "style": "IPY_MODEL_99556a31367c4a758b05df6af245bb0a",
            "value": " 818/818 [00:00&lt;00:00, 2011.00 examples/s]"
          }
        },
        "17133c6042294928b6a31b90bdd7386e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9ed8c128c0f54058a8e206dcf75695a3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7f15165f050140fcb999ac7fe86db8da": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f16cb1ea0e2f44e9af8b9023c827db93": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7e584fcd919642e2ae8546ca25ebbc6f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "33112d9b47e94bc3bee2603d1626317a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "99556a31367c4a758b05df6af245bb0a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MCCddTOfXy40",
        "outputId": "4ecb1c2c-a41f-416f-e449-24b11374b3f6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sat Aug 12 06:12:54 2023       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 525.105.17   Driver Version: 525.105.17   CUDA Version: 12.0     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   37C    P8     9W /  70W |      0MiB / 15360MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ]
        }
      ],
      "source": [
        "!nvidia-smi"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers[sentencepiece] datasets sacrebleu rouge_score py7zr -q"
      ],
      "metadata": {
        "id": "pLyiqDkJYBpJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install --upgrade accelerate\n",
        "!pip uninstall -y transformers accelerate\n",
        "!pip install transformers accelerate"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1znB_LqoYDw9",
        "outputId": "472214fc-2b02-44e4-889e-7060d7b9a242"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: accelerate in /usr/local/lib/python3.10/dist-packages (0.21.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from accelerate) (1.23.5)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (23.1)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate) (5.9.5)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from accelerate) (6.0.1)\n",
            "Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (2.0.1+cu118)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.12.2)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (4.7.1)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.1.2)\n",
            "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2.0.0)\n",
            "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.10.0->accelerate) (3.27.1)\n",
            "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.10.0->accelerate) (16.0.6)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.3)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\n",
            "Found existing installation: transformers 4.31.0\n",
            "Uninstalling transformers-4.31.0:\n",
            "  Successfully uninstalled transformers-4.31.0\n",
            "Found existing installation: accelerate 0.21.0\n",
            "Uninstalling accelerate-0.21.0:\n",
            "  Successfully uninstalled accelerate-0.21.0\n",
            "Collecting transformers\n",
            "  Using cached transformers-4.31.0-py3-none-any.whl (7.4 MB)\n",
            "Collecting accelerate\n",
            "  Using cached accelerate-0.21.0-py3-none-any.whl (244 kB)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.2)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.14.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.16.4)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.23.5)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.6.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.13.3)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.3.2)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.0)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate) (5.9.5)\n",
            "Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (2.0.1+cu118)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (2023.6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (4.7.1)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.1.2)\n",
            "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2.0.0)\n",
            "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.10.0->accelerate) (3.27.1)\n",
            "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.10.0->accelerate) (16.0.6)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.2.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2023.7.22)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.3)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\n",
            "Installing collected packages: transformers, accelerate\n",
            "Successfully installed accelerate-0.21.0 transformers-4.31.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import pipeline, set_seed\n",
        "from datasets import load_dataset, load_from_disk\n",
        "import matplotlib.pyplot as plt\n",
        "from datasets import load_dataset\n",
        "import pandas as pd\n",
        "from datasets import load_dataset, load_metric\n",
        "\n",
        "from transformers import AutoModelForSeq2SeqLM, AutoTokenizer\n",
        "\n",
        "import nltk\n",
        "from nltk.tokenize import sent_tokenize\n",
        "\n",
        "from tqdm import tqdm\n",
        "import torch\n",
        "\n",
        "nltk.download(\"punkt\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iq5aGkIZYFHI",
        "outputId": "c3974209-60ec-4c39-94c0-29e2ed24b183"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "from transformers import AutoModelForSeq2SeqLM, AutoTokenizer\n",
        "\n",
        "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
        "device"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "oGnpMNkgYmFl",
        "outputId": "523c53de-91e2-4c2d-d785-ed42daec0a13"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'cuda'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_ckpt = \"google/pegasus-cnn_dailymail\"\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_ckpt)\n",
        "\n",
        "model_pegasus = AutoModelForSeq2SeqLM.from_pretrained(model_ckpt).to(device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9Tg4P9_6YvkZ",
        "outputId": "28d90c74-a3d3-4b32-a07c-2b264a075671"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of PegasusForConditionalGeneration were not initialized from the model checkpoint at google/pegasus-cnn_dailymail and are newly initialized: ['model.encoder.embed_positions.weight', 'model.decoder.embed_positions.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import load_dataset\n",
        "\n",
        "dataset = load_dataset(\"samsum\")"
      ],
      "metadata": {
        "id": "uwUF6T7uYzcI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dataset"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uUbgfbTea5jY",
        "outputId": "17c41537-20a3-42ba-d222-6fce956e5e6c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['id', 'dialogue', 'summary'],\n",
              "        num_rows: 14732\n",
              "    })\n",
              "    test: Dataset({\n",
              "        features: ['id', 'dialogue', 'summary'],\n",
              "        num_rows: 819\n",
              "    })\n",
              "    validation: Dataset({\n",
              "        features: ['id', 'dialogue', 'summary'],\n",
              "        num_rows: 818\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dataset['train']"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cwKZ0L5-a8fF",
        "outputId": "7b884974-3160-4bd0-dc49-be18d54de439"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Dataset({\n",
              "    features: ['id', 'dialogue', 'summary'],\n",
              "    num_rows: 14732\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "split_lengths = [len(dataset[split])for split in dataset]\n",
        "\n",
        "print(f\"Split lengths: {split_lengths}\")\n",
        "print(f\"Features: {dataset['train'].column_names}\")\n",
        "print(\"\\nDialogue:\")\n",
        "\n",
        "print(dataset[\"test\"][1][\"dialogue\"])\n",
        "\n",
        "print(\"\\nSummary:\")\n",
        "\n",
        "print(dataset[\"test\"][1][\"summary\"])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZxklHAaEcCwU",
        "outputId": "192ed56b-d3ef-4a1e-a3a0-cd5a070c47e8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Split lengths: [14732, 819, 818]\n",
            "Features: ['id', 'dialogue', 'summary']\n",
            "\n",
            "Dialogue:\n",
            "Eric: MACHINE!\r\n",
            "Rob: That's so gr8!\r\n",
            "Eric: I know! And shows how Americans see Russian ;)\r\n",
            "Rob: And it's really funny!\r\n",
            "Eric: I know! I especially like the train part!\r\n",
            "Rob: Hahaha! No one talks to the machine like that!\r\n",
            "Eric: Is this his only stand-up?\r\n",
            "Rob: Idk. I'll check.\r\n",
            "Eric: Sure.\r\n",
            "Rob: Turns out no! There are some of his stand-ups on youtube.\r\n",
            "Eric: Gr8! I'll watch them now!\r\n",
            "Rob: Me too!\r\n",
            "Eric: MACHINE!\r\n",
            "Rob: MACHINE!\r\n",
            "Eric: TTYL?\r\n",
            "Rob: Sure :)\n",
            "\n",
            "Summary:\n",
            "Eric and Rob are going to watch a stand-up on youtube.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "def convert_examples_to_features(example_batch):\n",
        "    input_encodings = tokenizer(example_batch['dialogue'] , max_length = 1024, truncation = True )\n",
        "\n",
        "    with tokenizer.as_target_tokenizer():\n",
        "        target_encodings = tokenizer(example_batch['summary'], max_length = 128, truncation = True )\n",
        "\n",
        "    return {\n",
        "        'input_ids' : input_encodings['input_ids'],\n",
        "        'attention_mask': input_encodings['attention_mask'],\n",
        "        'labels': target_encodings['input_ids']\n",
        "    }"
      ],
      "metadata": {
        "id": "WX2UaSbQcSb1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "dataset_samsum_pt = dataset.map(convert_examples_to_features, batched = True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 105,
          "referenced_widgets": [
            "38bfa794ea0240e399ae1457405cc030",
            "9699415f3d9f409c9b00d049ab640869",
            "707e7b7ed859476d86b5729010b651af",
            "fc28eadb097d4371a36b78ae767082d7",
            "17133c6042294928b6a31b90bdd7386e",
            "9ed8c128c0f54058a8e206dcf75695a3",
            "7f15165f050140fcb999ac7fe86db8da",
            "f16cb1ea0e2f44e9af8b9023c827db93",
            "7e584fcd919642e2ae8546ca25ebbc6f",
            "33112d9b47e94bc3bee2603d1626317a",
            "99556a31367c4a758b05df6af245bb0a"
          ]
        },
        "id": "Q2NUbTZWcOrM",
        "outputId": "5b07a4bd-146d-493f-ffc8-94a0ba3b05c1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/818 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "38bfa794ea0240e399ae1457405cc030"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:3635: UserWarning: `as_target_tokenizer` is deprecated and will be removed in v5 of Transformers. You can tokenize your labels by using the argument `text_target` of the regular `__call__` method (either in the same call as your input texts if you use the same keyword arguments, or in a separate call.\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "dataset_samsum_pt[\"train\"]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j9nq-PwUd-GL",
        "outputId": "8790898b-59ce-40d2-920f-8b3179ddd038"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Dataset({\n",
              "    features: ['id', 'dialogue', 'summary', 'input_ids', 'attention_mask', 'labels'],\n",
              "    num_rows: 14732\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# Training\n",
        "\n",
        "from transformers import DataCollatorForSeq2Seq\n",
        "\n",
        "seq2seq_data_collator = DataCollatorForSeq2Seq(tokenizer, model=model_pegasus)"
      ],
      "metadata": {
        "id": "AMT68XwIeAd_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import TrainingArguments, Trainer\n",
        "\n",
        "trainer_args = TrainingArguments(\n",
        "    output_dir='pegasus-samsum', num_train_epochs=2, warmup_steps=500,\n",
        "    per_device_train_batch_size=1, per_device_eval_batch_size=1,\n",
        "    weight_decay=0.01, logging_steps=10,\n",
        "    evaluation_strategy='steps', eval_steps=500, save_steps=1e6,\n",
        "    gradient_accumulation_steps=16,\n",
        ")"
      ],
      "metadata": {
        "id": "-noj65yBeEAA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "trainer = Trainer(model=model_pegasus, args=trainer_args,\n",
        "                  tokenizer=tokenizer, data_collator=seq2seq_data_collator,\n",
        "                  train_dataset=dataset_samsum_pt[\"train\"],\n",
        "                  eval_dataset=dataset_samsum_pt[\"validation\"])"
      ],
      "metadata": {
        "id": "iSt4rU8meZMv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "trainer.train()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 280
        },
        "id": "yiTi80hMeeOp",
        "outputId": "0aa38eda-46e6-48e1-df97-35e4e0ba4908"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  warnings.warn(\n",
            "You're using a PegasusTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='1840' max='1840' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [1840/1840 1:35:20, Epoch 1/2]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>500</td>\n",
              "      <td>1.629700</td>\n",
              "      <td>1.485818</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1000</td>\n",
              "      <td>1.573500</td>\n",
              "      <td>1.404008</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1500</td>\n",
              "      <td>1.344700</td>\n",
              "      <td>1.380824</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=1840, training_loss=1.642336044104203, metrics={'train_runtime': 5723.8795, 'train_samples_per_second': 5.148, 'train_steps_per_second': 0.321, 'total_flos': 1.105515010535424e+16, 'train_loss': 1.642336044104203, 'epoch': 2.0})"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Evaluation\n",
        "\n",
        "def generate_batch_sized_chunks(list_of_elements, batch_size):\n",
        "    \"\"\"split the dataset into smaller batches that we can process simultaneously\n",
        "    Yield successive batch-sized chunks from list_of_elements.\"\"\"\n",
        "    for i in range(0, len(list_of_elements), batch_size):\n",
        "        yield list_of_elements[i : i + batch_size]"
      ],
      "metadata": {
        "id": "3PIr6OraegMd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def calculate_metric_on_test_ds(dataset, metric, model, tokenizer,\n",
        "                               batch_size=16, device=device,\n",
        "                               column_text=\"article\",\n",
        "                               column_summary=\"highlights\"):\n",
        "    article_batches = list(generate_batch_sized_chunks(dataset[column_text], batch_size))\n",
        "    target_batches = list(generate_batch_sized_chunks(dataset[column_summary], batch_size))\n",
        "\n",
        "    for article_batch, target_batch in tqdm(\n",
        "        zip(article_batches, target_batches), total=len(article_batches)):\n",
        "\n",
        "        inputs = tokenizer(article_batch, max_length=1024,  truncation=True,\n",
        "                        padding=\"max_length\", return_tensors=\"pt\")\n",
        "\n",
        "        summaries = model.generate(input_ids=inputs[\"input_ids\"].to(device),\n",
        "                         attention_mask=inputs[\"attention_mask\"].to(device),\n",
        "                         length_penalty=0.8, num_beams=8, max_length=128)\n",
        "        ''' parameter for length penalty ensures that the model does not generate sequences that are too long. '''\n",
        "\n",
        "        # Finally, we decode the generated texts,\n",
        "        # replace the  token, and add the decoded texts with the references to the metric.\n",
        "        decoded_summaries = [tokenizer.decode(s, skip_special_tokens=True,\n",
        "                                clean_up_tokenization_spaces=True)\n",
        "               for s in summaries]\n",
        "\n",
        "        decoded_summaries = [d.replace(\"\", \" \") for d in decoded_summaries]\n",
        "\n",
        "\n",
        "        metric.add_batch(predictions=decoded_summaries, references=target_batch)\n",
        "\n",
        "    #  Finally compute and return the ROUGE scores.\n",
        "    score = metric.compute()\n",
        "    return score"
      ],
      "metadata": {
        "id": "3AH6-p7SfAFu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "rouge_names = [\"rouge1\", \"rouge2\", \"rougeL\", \"rougeLsum\"]\n",
        "rouge_metric = load_metric('rouge')"
      ],
      "metadata": {
        "id": "vLC-IH5KeXCp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "score = calculate_metric_on_test_ds(\n",
        "    dataset['test'][0:10], rouge_metric, trainer.model, tokenizer, batch_size = 2, column_text = 'dialogue', column_summary= 'summary'\n",
        ")\n",
        "\n",
        "rouge_dict = dict((rn, score[rn].mid.fmeasure ) for rn in rouge_names)\n",
        "\n",
        "pd.DataFrame(rouge_dict, index = [f'pegasus'] )"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 98
        },
        "id": "Ndnm1XAbb_zv",
        "outputId": "42531863-1710-4be9-baa0-9cbbd49b754f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 5/5 [00:16<00:00,  3.25s/it]\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "           rouge1  rouge2    rougeL  rougeLsum\n",
              "pegasus  0.024595     0.0  0.024311   0.024418"
            ],
            "text/html": [
              "\n",
              "\n",
              "  <div id=\"df-426bfe85-f14e-4634-9c26-5ffb987af314\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>rouge1</th>\n",
              "      <th>rouge2</th>\n",
              "      <th>rougeL</th>\n",
              "      <th>rougeLsum</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>pegasus</th>\n",
              "      <td>0.024595</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.024311</td>\n",
              "      <td>0.024418</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-426bfe85-f14e-4634-9c26-5ffb987af314')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "\n",
              "\n",
              "\n",
              "    <div id=\"df-cbda3610-7c93-48d8-b2d0-4ec28783807a\">\n",
              "      <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-cbda3610-7c93-48d8-b2d0-4ec28783807a')\"\n",
              "              title=\"Suggest charts.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "      </button>\n",
              "    </div>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "    <script>\n",
              "      async function quickchart(key) {\n",
              "        const containerElement = document.querySelector('#' + key);\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      }\n",
              "    </script>\n",
              "\n",
              "      <script>\n",
              "\n",
              "function displayQuickchartButton(domScope) {\n",
              "  let quickchartButtonEl =\n",
              "    domScope.querySelector('#df-cbda3610-7c93-48d8-b2d0-4ec28783807a button.colab-df-quickchart');\n",
              "  quickchartButtonEl.style.display =\n",
              "    google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "}\n",
              "\n",
              "        displayQuickchartButton(document);\n",
              "      </script>\n",
              "      <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-426bfe85-f14e-4634-9c26-5ffb987af314 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-426bfe85-f14e-4634-9c26-5ffb987af314');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "## Save model\n",
        "model_pegasus.save_pretrained(\"pegasus-samsum-model\")"
      ],
      "metadata": {
        "id": "4JfcEf-OfRVU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "## Save tokenizer\n",
        "tokenizer.save_pretrained(\"tokenizer\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MNYNfDgCfTyr",
        "outputId": "170d5a8f-5323-41e8-dc01-972bb8bbbae2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('tokenizer/tokenizer_config.json',\n",
              " 'tokenizer/special_tokens_map.json',\n",
              " 'tokenizer/spiece.model',\n",
              " 'tokenizer/added_tokens.json',\n",
              " 'tokenizer/tokenizer.json')"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Load\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"/content/tokenizer\")"
      ],
      "metadata": {
        "id": "CehYpfGTfVd2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Prediction\n",
        "\n",
        "gen_kwargs = {\"length_penalty\": 0.8, \"num_beams\":8, \"max_length\": 128}\n",
        "\n",
        "\n",
        "\n",
        "sample_text = dataset[\"test\"][0][\"dialogue\"]\n",
        "\n",
        "reference = dataset[\"test\"][0][\"summary\"]\n",
        "\n",
        "pipe = pipeline(\"summarization\", model=\"pegasus-samsum-model\",tokenizer=tokenizer)\n",
        "\n",
        "##\n",
        "print(\"Dialogue:\")\n",
        "print(sample_text)\n",
        "\n",
        "\n",
        "print(\"\\nReference Summary:\")\n",
        "print(reference)\n",
        "\n",
        "\n",
        "print(\"\\nModel Summary:\")\n",
        "print(pipe(sample_text, **gen_kwargs)[0][\"summary_text\"])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hHuv1Cx8fWw3",
        "outputId": "b61f8347-d980-43c5-90b1-3606c9e05075"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Your max_length is set to 128, but your input_length is only 122. Since this is a summarization task, where outputs shorter than the input are typically wanted, you might consider decreasing max_length manually, e.g. summarizer('...', max_length=61)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Dialogue:\n",
            "Hannah: Hey, do you have Betty's number?\n",
            "Amanda: Lemme check\n",
            "Hannah: <file_gif>\n",
            "Amanda: Sorry, can't find it.\n",
            "Amanda: Ask Larry\n",
            "Amanda: He called her last time we were at the park together\n",
            "Hannah: I don't know him well\n",
            "Hannah: <file_gif>\n",
            "Amanda: Don't be shy, he's very nice\n",
            "Hannah: If you say so..\n",
            "Hannah: I'd rather you texted him\n",
            "Amanda: Just text him 🙂\n",
            "Hannah: Urgh.. Alright\n",
            "Hannah: Bye\n",
            "Amanda: Bye bye\n",
            "\n",
            "Reference Summary:\n",
            "Hannah needs Betty's number but Amanda doesn't have it. She needs to contact Larry.\n",
            "\n",
            "Model Summary:\n",
            "Amanda can't find Betty's number. Larry called her last time they were at the park together. Hannah wants Amanda to text him instead of finding Betty's number.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "YSoKtsv7fYVr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "JGC_WlZaYl9N"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "bX3l8shKYMPf"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}