WebScraping.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qqrf_BJQVQ-S",
        "outputId": "a7650461-056e-4e18-b55f-850dda581fa4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[1;31mE: \u001b[0mCould not open lock file /var/lib/dpkg/lock-frontend - open (13: Permission denied)\u001b[0m\n",
            "\u001b[1;31mE: \u001b[0mUnable to acquire the dpkg frontend lock (/var/lib/dpkg/lock-frontend), are you root?\u001b[0m\n",
            "\u001b[1;31mE: \u001b[0mCould not open lock file /var/lib/dpkg/lock-frontend - open (13: Permission denied)\u001b[0m\n",
            "\u001b[1;31mE: \u001b[0mUnable to acquire the dpkg frontend lock (/var/lib/dpkg/lock-frontend), are you root?\u001b[0m\n",
            "Requirement already satisfied: pytesseract in /home/vm/anaconda3/lib/python3.11/site-packages (0.3.10)\n",
            "Requirement already satisfied: packaging>=21.3 in /home/vm/anaconda3/lib/python3.11/site-packages (from pytesseract) (23.1)\n",
            "Requirement already satisfied: Pillow>=8.0.0 in /home/vm/anaconda3/lib/python3.11/site-packages (from pytesseract) (10.2.0)\n"
          ]
        }
      ],
      "source": [
        "!apt install tesseract-ocr\n",
        "!apt install libtesseract-dev\n",
        "!pip install pytesseract"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "fIMP6AG0MqWp"
      },
      "outputs": [],
      "source": [
        "from bs4 import BeautifulSoup\n",
        "import pandas as pd\n",
        "import json\n",
        "import requests\n",
        "from PIL import Image\n",
        "import pytesseract\n",
        "import numpy as np\n",
        "import requests\n",
        "from io import BytesIO"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZmQB2thgMrgG"
      },
      "source": [
        "# Dataset\n",
        "\n",
        "To obtain the dataset we have done web scrapping in the https://www.poetryfoundation.org/ web. First we obtain the number of poems of a given period and then we iterate over all the pages to obtain the title, author and the poem itself."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "9mXBKwO2MoNt"
      },
      "outputs": [],
      "source": [
        "def extract_total_results(url):\n",
        "    response = requests.get(url)\n",
        "    data = response.json()\n",
        "    total_results = data['TotalResults']\n",
        "    return total_results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "r830gp6oBF7N"
      },
      "outputs": [],
      "source": [
        "def extract_text_from_image(url):\n",
        "  try:\n",
        "        response = requests.get(url)\n",
        "        img_bytes = BytesIO(response.content)\n",
        "        img1 = np.array(Image.open(img_bytes))\n",
        "        text = pytesseract.image_to_string(img1)\n",
        "        text = text.replace('\\n', ' ')\n",
        "        return text\n",
        "  except Exception as e:\n",
        "        print(f\"An error occurred: {e}\")\n",
        "        return None"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XdfzagF9JdDe",
        "outputId": "f97f06e8-ebbf-47aa-ca98-a34e90f8ee21"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "https://static.poetryfoundation.org/jstor/i20585549/pages/14.png\n",
            "POETRY  three sonnets of ronsard  COMME ON VOIT SUR LA BRANCHE  As one sees on its branch, in May, the rose  In its bright youth and new astonishment,  Making the heavens jealous of its tint  When, touched with dew, inthe first light it glows, Then, in its petal, grace and love repose,  Filling the gardens and the trees with scent;  But, scourged by the sun’s heat, by the rain bent, Drooping, it dies: petal from petal blows.  So your bright youth, which earth and heaven adore, Lies now in ashes, for no love can strive  Against the murdering Fate. For obsequy,  Receive my tears, this vase of milk I pour,  This basket full of flowers, so that alive  And dead, your body shall all roses be.  200 \n"
          ]
        }
      ],
      "source": [
        "url = 'https://www.poetryfoundation.org/poetrymagazine/browse?contentId=26576'\n",
        "\n",
        "response = requests.get(url)\n",
        "poem_soup=BeautifulSoup(response.text, 'html.parser')\n",
        "poem=poem_soup.find(class_='c-assetStack-media')\n",
        "img = poem.find('img')\n",
        "src_value = img['src']\n",
        "print(src_value)\n",
        "text=extract_text_from_image(src_value)\n",
        "\n",
        "print(text)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "LEsWAdrBEyF2"
      },
      "outputs": [],
      "source": [
        "def extract_poems(url,period):\n",
        "    response = requests.get(url)\n",
        "    data = response.json()\n",
        "\n",
        "    # Extract attributes from each entry\n",
        "    entries = []\n",
        "    for entry in data['Entries']:\n",
        "        entry_data = {\n",
        "            'id': entry['id'],\n",
        "            'title': entry['title'],\n",
        "            'author': entry['author'],\n",
        "            'snippet': entry['snippet'],\n",
        "            'link': entry['link'],\n",
        "            'categories': [category['title'] for category in entry['categories']],\n",
        "            'period': period\n",
        "        }\n",
        "        \n",
        "        # Fetch the poem content from the link\n",
        "        poem_response = requests.get(entry_data['link'])\n",
        "        poem_soup = BeautifulSoup(poem_response.text, 'html.parser')\n",
        "        poem = poem_soup.find(class_='o-poem')\n",
        "        if poem != None:\n",
        "          try:\n",
        "            poem_text = poem.get_text(separator=' ', strip=True) # If we wanted to take into accouent the form of the poem use \\n as separator\n",
        "          except Exception as e:\n",
        "            print(f\"An error occurred: {e}\")\n",
        "            poem_text = None\n",
        "          # Add poem text to the entry data\n",
        "          entry_data['poem'] = poem_text\n",
        "        else: # When the poem is an image\n",
        "          poem=poem_soup.find(class_='c-assetStack-media')\n",
        "          if poem != None: # If there is a class c-assetStack-media\n",
        "            img = poem.find('img')\n",
        "            if img == None: # If the image is not found\n",
        "              entry_data['poem'] = None\n",
        "            else: # If the image is found\n",
        "              src_value = img['src']\n",
        "              poem_text = extract_text_from_image(src_value)\n",
        "              entry_data['poem'] = poem_text\n",
        "          else: # If there is no class c-assetStack-media, this means that the poem is an audio or other data types\n",
        "            entry_data['poem'] = None\n",
        "\n",
        "        entries.append(entry_data)\n",
        "\n",
        "    # Create a pandas DataFrame\n",
        "    df = pd.DataFrame(entries)\n",
        "    return df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "HB3bDvoGV89G"
      },
      "outputs": [],
      "source": [
        "# Initialize an empty list to store individual DataFrames\n",
        "dfs = []\n",
        "period_ids = {\n",
        "    'Middle English': 158,\n",
        "    'Augustan': 149,\n",
        "    'Renaissance': 163,\n",
        "    'Romantic': 164,\n",
        "    'Victorian': 165,\n",
        "    'Fugitive': 153,\n",
        "    'Georgian': 154,\n",
        "    'Harlem Renaissance': 155,\n",
        "    'Imagist': 156,\n",
        "    'Modern': 159,\n",
        "    'Objectivist': 162,\n",
        "    'Beat': 150,\n",
        "    'Black Arts Movement': 304,\n",
        "    'Black Mountain': 151,\n",
        "    'Confessional': 152,\n",
        "    'Language Poetry': 157,\n",
        "    'New York School': 160,\n",
        "    'New York School (2nd Generation)': 161\n",
        "}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SjTHwJ4-E9HY",
        "outputId": "2c865096-3ff0-4468-f2ba-f1e3bdf3f727"
      },
      "outputs": [],
      "source": [
        "# #Loop over all the periods to find how many entries they have\n",
        "for i in period_ids:\n",
        "  total_results = extract_total_results(\"https://www.poetryfoundation.org/ajax/poems?page=1&sort_by=title&school-period={}\".format(period_ids[i]))\n",
        "  print(\"Extracting poems of \", i)\n",
        "  # Loop through the URLs\n",
        "  for j in range(total_results // 20 + 1): # 20 poems per page\n",
        "      url = \"https://www.poetryfoundation.org/ajax/poems?page={}&sort_by=title&school-period={}\".format(j+1, period_ids[i])\n",
        "      # Extract DataFrame from each URL and append to the list\n",
        "      dfs.append(extract_poems(url,i))\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "E9n-6hLfbqpn"
      },
      "outputs": [],
      "source": [
        "# # Concatenate all DataFrames into a single DataFrame\n",
        "final_df = pd.concat(dfs, ignore_index=True)\n",
        "\n",
        "# Display the final DataFrame\n",
        "print(len(final_df))\n",
        "final_df.head()\n",
        "\n",
        "final_df.to_csv(\"poems_data.csv\", index=False, quoting=1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Full Poems Extraction for good Tokenization"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {},
      "outputs": [],
      "source": [
        "def extract_poems(url):\n",
        "    response = requests.get(url)\n",
        "    data = response.json()\n",
        "\n",
        "    # Extract attributes from each entry\n",
        "    entries = []\n",
        "    for entry in data['Entries']:\n",
        "        entry_data = {\n",
        "            'id': entry['id'],\n",
        "            'title': entry['title'],\n",
        "            'author': entry['author'],\n",
        "            'snippet': entry['snippet'],\n",
        "            'link': entry['link'],\n",
        "            'categories': [category['title'] for category in entry['categories']]            \n",
        "        }\n",
        "\n",
        "        # Fetch the poem content from the link\n",
        "        poem_response = requests.get(entry_data['link'])\n",
        "        poem_soup = BeautifulSoup(poem_response.text, 'html.parser')\n",
        "        poem = poem_soup.find(class_='o-poem')\n",
        "        if poem != None:\n",
        "          try:\n",
        "            poem_text = poem.get_text(separator=' ', strip=True) # If we wanted to take into accouent the form of the poem use \\n as separator\n",
        "          except Exception as e:\n",
        "            print(f\"An error occurred: {e}\")\n",
        "            poem_text = None\n",
        "          # Add poem text to the entry data\n",
        "          entry_data['poem'] = poem_text\n",
        "        else: # When the poem is an image\n",
        "          poem=poem_soup.find(class_='c-assetStack-media')\n",
        "          if poem != None: # If there is a class c-assetStack-media\n",
        "            img = poem.find('img')\n",
        "            if img == None: # If the image is not found\n",
        "              entry_data['poem'] = None\n",
        "            else: # If the image is found\n",
        "              src_value = img['src']\n",
        "              poem_text = extract_text_from_image(src_value)\n",
        "              entry_data['poem'] = poem_text\n",
        "          else: # If there is no class c-assetStack-media, this means that the poem is an audio or other data types\n",
        "            entry_data['poem'] = None\n",
        "\n",
        "        entries.append(entry_data)\n",
        "\n",
        "    # Create a pandas DataFrame\n",
        "    df = pd.DataFrame(entries)\n",
        "    return df\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "We have counted already  100 , 32.248294830322266 seconds have passed.\n",
            "We have counted already  200 , 26.4195818901062 seconds have passed.\n",
            "We have counted already  300 , 31.873921632766724 seconds have passed.\n",
            "We have counted already  400 , 56.13200354576111 seconds have passed.\n",
            "We have counted already  500 , 58.59949707984924 seconds have passed.\n",
            "We have counted already  600 , 57.18841743469238 seconds have passed.\n",
            "We have counted already  700 , 57.89660143852234 seconds have passed.\n",
            "We have counted already  800 , 64.29464888572693 seconds have passed.\n",
            "We have counted already  900 , 58.9615204334259 seconds have passed.\n",
            "We have counted already  1000 , 62.31735610961914 seconds have passed.\n",
            "We have counted already  1100 , 58.395796060562134 seconds have passed.\n",
            "We have counted already  1200 , 59.80884552001953 seconds have passed.\n",
            "We have counted already  1300 , 57.53302621841431 seconds have passed.\n",
            "We have counted already  1400 , 58.98824191093445 seconds have passed.\n",
            "We have counted already  1500 , 60.97970366477966 seconds have passed.\n",
            "We have counted already  1600 , 54.82931423187256 seconds have passed.\n",
            "We have counted already  1700 , 61.768121004104614 seconds have passed.\n",
            "We have counted already  1800 , 60.01085138320923 seconds have passed.\n",
            "We have counted already  1900 , 59.91727137565613 seconds have passed.\n",
            "We have counted already  2000 , 59.63112759590149 seconds have passed.\n",
            "We have counted already  2100 , 59.3179612159729 seconds have passed.\n",
            "We have counted already  2200 , 56.09359574317932 seconds have passed.\n",
            "We have counted already  2300 , 60.50134634971619 seconds have passed.\n",
            "We have counted already  2400 , 60.49095559120178 seconds have passed.\n",
            "We have counted already  2500 , 56.31525135040283 seconds have passed.\n",
            "We have counted already  2600 , 58.79864048957825 seconds have passed.\n",
            "We have counted already  2700 , 57.501044273376465 seconds have passed.\n",
            "We have counted already  2800 , 59.18732786178589 seconds have passed.\n",
            "We have counted already  2900 , 54.87829065322876 seconds have passed.\n",
            "We have counted already  3000 , 59.16655373573303 seconds have passed.\n",
            "We have counted already  3100 , 56.81876468658447 seconds have passed.\n",
            "We have counted already  3200 , 58.384538412094116 seconds have passed.\n",
            "We have counted already  3300 , 58.86522960662842 seconds have passed.\n",
            "We have counted already  3400 , 58.53689765930176 seconds have passed.\n",
            "We have counted already  3500 , 60.494866371154785 seconds have passed.\n",
            "We have counted already  3600 , 58.24710536003113 seconds have passed.\n",
            "We have counted already  3700 , 60.689777851104736 seconds have passed.\n",
            "We have counted already  3800 , 62.426334381103516 seconds have passed.\n",
            "We have counted already  3900 , 59.043254375457764 seconds have passed.\n",
            "We have counted already  4000 , 59.999377727508545 seconds have passed.\n",
            "We have counted already  4100 , 58.198509216308594 seconds have passed.\n",
            "We have counted already  4200 , 57.52655625343323 seconds have passed.\n",
            "We have counted already  4300 , 59.99922704696655 seconds have passed.\n",
            "We have counted already  4400 , 57.27505445480347 seconds have passed.\n",
            "We have counted already  4500 , 58.802542209625244 seconds have passed.\n",
            "We have counted already  4600 , 61.177424907684326 seconds have passed.\n",
            "An error occurred: cannot identify image file <_io.BytesIO object at 0x7f944c9e80e0>\n",
            "We have counted already  4700 , 59.84782433509827 seconds have passed.\n",
            "We have counted already  4800 , 51.49475169181824 seconds have passed.\n",
            "We have counted already  4900 , 56.789695501327515 seconds have passed.\n",
            "We have counted already  5000 , 60.48564910888672 seconds have passed.\n",
            "We have counted already  5100 , 59.7560179233551 seconds have passed.\n",
            "We have counted already  5200 , 60.779104232788086 seconds have passed.\n",
            "We have counted already  5300 , 59.15512251853943 seconds have passed.\n",
            "We have counted already  5400 , 60.18132472038269 seconds have passed.\n"
          ]
        }
      ],
      "source": [
        "import time\n",
        "total_results = extract_total_results(\"https://www.poetryfoundation.org/ajax/poems?page=1\")\n",
        "dfs = []\n",
        "n_poems = 0\n",
        "count =0\n",
        "\n",
        "for j in range(2101,total_results // 20 + 1): # 20 poems per page\n",
        "    start_time = time.time()\n",
        "\n",
        "    url = \"https://www.poetryfoundation.org/ajax/poems?page={}\".format(j+1)\n",
        "    # Extract DataFrame from each URL and append to the list\n",
        "    dfs.append(extract_poems(url))\n",
        "\n",
        "    n_poems += 20\n",
        "\n",
        "    count += 1\n",
        "    if count == 5:\n",
        "        print(\"We have counted already \", n_poems, \",\", (time.time() - start_time), \"seconds have passed.\")\n",
        "        count = 0\n",
        "        start_time = 0"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "5451\n"
          ]
        }
      ],
      "source": [
        "# Concatenate all DataFrames into a single DataFrame\n",
        "df = pd.concat(dfs, ignore_index=True)\n",
        "\n",
        "# Display the final DataFrame\n",
        "print(len(df))\n",
        "df.head()\n",
        "\n",
        "df.to_csv(\"full_poems_data.csv\", index=False, quoting=1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "<div style=\"background-color:green; padding:10px\">\n",
        "Notice that we have been iterating through this notebook as it took a lot of time to process everything. We have been merging datasets in the file named \"Merging_Datasets\"\n",
        "</div>\n"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.11.7"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}