Lab 2/Lab_2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "6ced2d94",
   "metadata": {},
   "source": [
    "# Tokenization by Built-in function split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "94dfb575",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Tokenization', 'by', 'Built-in', 'function', 'split']\n"
     ]
    }
   ],
   "source": [
    "sentence = \"Tokenization by Built-in function split\"\n",
    "tokens = sentence.split()\n",
    "print(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "882e7c3e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['This', ' is an example sentence.']\n"
     ]
    }
   ],
   "source": [
    "sentence = \"This, is an example sentence.\"\n",
    "tokens = sentence.split(',')\n",
    "print(tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9579f30f",
   "metadata": {},
   "source": [
    "# Tokenization using TextBlob Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9654b615",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "**********************************************************************\n",
      "  Resource \u001b[93mpunkt\u001b[0m not found.\n",
      "  Please use the NLTK Downloader to obtain the resource:\n",
      "\n",
      "  \u001b[31m>>> import nltk\n",
      "  >>> nltk.download('punkt')\n",
      "  \u001b[0m\n",
      "  For more information see: https://www.nltk.org/data.html\n",
      "\n",
      "  Attempted to load \u001b[93mtokenizers/punkt/english.pickle\u001b[0m\n",
      "\n",
      "  Searched in:\n",
      "    - 'C:\\\\Users\\\\vivek/nltk_data'\n",
      "    - 'C:\\\\Users\\\\vivek\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python310\\\\nltk_data'\n",
      "    - 'C:\\\\Users\\\\vivek\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python310\\\\share\\\\nltk_data'\n",
      "    - 'C:\\\\Users\\\\vivek\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python310\\\\lib\\\\nltk_data'\n",
      "    - 'C:\\\\Users\\\\vivek\\\\AppData\\\\Roaming\\\\nltk_data'\n",
      "    - 'C:\\\\nltk_data'\n",
      "    - 'D:\\\\nltk_data'\n",
      "    - 'E:\\\\nltk_data'\n",
      "    - ''\n",
      "**********************************************************************\n",
      "\n"
     ]
    },
    {
     "ename": "MissingCorpusError",
     "evalue": "\nLooks like you are missing some required data for this feature.\n\nTo download the necessary data, simply run\n\n    python -m textblob.download_corpora\n\nor use the NLTK downloader to download the missing data: http://nltk.org/data.html\nIf this doesn't fix the problem, file an issue at https://github.com/sloria/TextBlob/issues.\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mLookupError\u001b[0m                               Traceback (most recent call last)",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\textblob\\decorators.py:35\u001b[0m, in \u001b[0;36mrequires_nltk_corpus.<locals>.decorated\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     34\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m---> 35\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m func(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m     36\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mLookupError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\textblob\\tokenizers.py:57\u001b[0m, in \u001b[0;36mSentenceTokenizer.tokenize\u001b[1;34m(self, text)\u001b[0m\n\u001b[0;32m     56\u001b[0m \u001b[38;5;124;03m'''Return a list of sentences.'''\u001b[39;00m\n\u001b[1;32m---> 57\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mnltk\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtokenize\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msent_tokenize\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtext\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\nltk\\tokenize\\__init__.py:106\u001b[0m, in \u001b[0;36msent_tokenize\u001b[1;34m(text, language)\u001b[0m\n\u001b[0;32m     97\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m     98\u001b[0m \u001b[38;5;124;03mReturn a sentence-tokenized copy of *text*,\u001b[39;00m\n\u001b[0;32m     99\u001b[0m \u001b[38;5;124;03musing NLTK's recommended sentence tokenizer\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    104\u001b[0m \u001b[38;5;124;03m:param language: the model name in the Punkt corpus\u001b[39;00m\n\u001b[0;32m    105\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m--> 106\u001b[0m tokenizer \u001b[38;5;241m=\u001b[39m \u001b[43mload\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43mf\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtokenizers/punkt/\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mlanguage\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[38;5;124;43m.pickle\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[0;32m    107\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m tokenizer\u001b[38;5;241m.\u001b[39mtokenize(text)\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\nltk\\data.py:750\u001b[0m, in \u001b[0;36mload\u001b[1;34m(resource_url, format, cache, verbose, logic_parser, fstruct_reader, encoding)\u001b[0m\n\u001b[0;32m    749\u001b[0m \u001b[38;5;66;03m# Load the resource.\u001b[39;00m\n\u001b[1;32m--> 750\u001b[0m opened_resource \u001b[38;5;241m=\u001b[39m \u001b[43m_open\u001b[49m\u001b[43m(\u001b[49m\u001b[43mresource_url\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    752\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mformat\u001b[39m \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mraw\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\nltk\\data.py:876\u001b[0m, in \u001b[0;36m_open\u001b[1;34m(resource_url)\u001b[0m\n\u001b[0;32m    875\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m protocol \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mor\u001b[39;00m protocol\u001b[38;5;241m.\u001b[39mlower() \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnltk\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m--> 876\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfind\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath_\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpath\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mopen()\n\u001b[0;32m    877\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m protocol\u001b[38;5;241m.\u001b[39mlower() \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfile\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m    878\u001b[0m     \u001b[38;5;66;03m# urllib might not use mode='rb', so handle this one ourselves:\u001b[39;00m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\nltk\\data.py:583\u001b[0m, in \u001b[0;36mfind\u001b[1;34m(resource_name, paths)\u001b[0m\n\u001b[0;32m    582\u001b[0m resource_not_found \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00msep\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00mmsg\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00msep\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m--> 583\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mLookupError\u001b[39;00m(resource_not_found)\n",
      "\u001b[1;31mLookupError\u001b[0m: \n**********************************************************************\n  Resource \u001b[93mpunkt\u001b[0m not found.\n  Please use the NLTK Downloader to obtain the resource:\n\n  \u001b[31m>>> import nltk\n  >>> nltk.download('punkt')\n  \u001b[0m\n  For more information see: https://www.nltk.org/data.html\n\n  Attempted to load \u001b[93mtokenizers/punkt/english.pickle\u001b[0m\n\n  Searched in:\n    - 'C:\\\\Users\\\\vivek/nltk_data'\n    - 'C:\\\\Users\\\\vivek\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python310\\\\nltk_data'\n    - 'C:\\\\Users\\\\vivek\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python310\\\\share\\\\nltk_data'\n    - 'C:\\\\Users\\\\vivek\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python310\\\\lib\\\\nltk_data'\n    - 'C:\\\\Users\\\\vivek\\\\AppData\\\\Roaming\\\\nltk_data'\n    - 'C:\\\\nltk_data'\n    - 'D:\\\\nltk_data'\n    - 'E:\\\\nltk_data'\n    - ''\n**********************************************************************\n",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mMissingCorpusError\u001b[0m                        Traceback (most recent call last)",
      "Input \u001b[1;32mIn [9]\u001b[0m, in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      3\u001b[0m text \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mThis is an example sentence. This is another one.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m      4\u001b[0m blob \u001b[38;5;241m=\u001b[39m TextBlob(text)\n\u001b[1;32m----> 5\u001b[0m sentences \u001b[38;5;241m=\u001b[39m \u001b[43mblob\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msentences\u001b[49m\n\u001b[0;32m      6\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m sentence \u001b[38;5;129;01min\u001b[39;00m sentences:\n\u001b[0;32m      7\u001b[0m     tokens \u001b[38;5;241m=\u001b[39m sentence\u001b[38;5;241m.\u001b[39mwords\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\textblob\\decorators.py:24\u001b[0m, in \u001b[0;36mcached_property.__get__\u001b[1;34m(self, obj, cls)\u001b[0m\n\u001b[0;32m     22\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m obj \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m     23\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\n\u001b[1;32m---> 24\u001b[0m value \u001b[38;5;241m=\u001b[39m obj\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__dict__\u001b[39m[\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfunc\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43mobj\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     25\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m value\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\textblob\\blob.py:668\u001b[0m, in \u001b[0;36mTextBlob.sentences\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    665\u001b[0m \u001b[38;5;129m@cached_property\u001b[39m\n\u001b[0;32m    666\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21msentences\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m    667\u001b[0m     \u001b[38;5;124;03m\"\"\"Return list of :class:`Sentence <Sentence>` objects.\"\"\"\u001b[39;00m\n\u001b[1;32m--> 668\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_create_sentence_objects\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\textblob\\blob.py:712\u001b[0m, in \u001b[0;36mTextBlob._create_sentence_objects\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    709\u001b[0m \u001b[38;5;124;03m'''Returns a list of Sentence objects from the raw text.\u001b[39;00m\n\u001b[0;32m    710\u001b[0m \u001b[38;5;124;03m'''\u001b[39;00m\n\u001b[0;32m    711\u001b[0m sentence_objects \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m--> 712\u001b[0m sentences \u001b[38;5;241m=\u001b[39m \u001b[43msent_tokenize\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mraw\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    713\u001b[0m char_index \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m  \u001b[38;5;66;03m# Keeps track of character index within the blob\u001b[39;00m\n\u001b[0;32m    714\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m sent \u001b[38;5;129;01min\u001b[39;00m sentences:\n\u001b[0;32m    715\u001b[0m     \u001b[38;5;66;03m# Compute the start and end indices of the sentence\u001b[39;00m\n\u001b[0;32m    716\u001b[0m     \u001b[38;5;66;03m# within the blob\u001b[39;00m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\textblob\\base.py:64\u001b[0m, in \u001b[0;36mBaseTokenizer.itokenize\u001b[1;34m(self, text, *args, **kwargs)\u001b[0m\n\u001b[0;32m     57\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mitokenize\u001b[39m(\u001b[38;5;28mself\u001b[39m, text, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m     58\u001b[0m     \u001b[38;5;124;03m\"\"\"Return a generator that generates tokens \"on-demand\".\u001b[39;00m\n\u001b[0;32m     59\u001b[0m \n\u001b[0;32m     60\u001b[0m \u001b[38;5;124;03m    .. versionadded:: 0.6.0\u001b[39;00m\n\u001b[0;32m     61\u001b[0m \n\u001b[0;32m     62\u001b[0m \u001b[38;5;124;03m    :rtype: generator\u001b[39;00m\n\u001b[0;32m     63\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[1;32m---> 64\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m (t \u001b[38;5;28;01mfor\u001b[39;00m t \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtokenize(text, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs))\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\textblob\\decorators.py:38\u001b[0m, in \u001b[0;36mrequires_nltk_corpus.<locals>.decorated\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     36\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mLookupError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n\u001b[0;32m     37\u001b[0m     \u001b[38;5;28mprint\u001b[39m(err)\n\u001b[1;32m---> 38\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m MissingCorpusError()\n",
      "\u001b[1;31mMissingCorpusError\u001b[0m: \nLooks like you are missing some required data for this feature.\n\nTo download the necessary data, simply run\n\n    python -m textblob.download_corpora\n\nor use the NLTK downloader to download the missing data: http://nltk.org/data.html\nIf this doesn't fix the problem, file an issue at https://github.com/sloria/TextBlob/issues.\n"
     ]
    }
   ],
   "source": [
    "from textblob import TextBlob\n",
    "\n",
    "text = \"This is an example sentence. This is another one.\"\n",
    "blob = TextBlob(text)\n",
    "sentences = blob.sentences\n",
    "for sentence in sentences:\n",
    "    tokens = sentence.words\n",
    "    print(tokens)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "80c9fb63",
   "metadata": {},
   "source": [
    "# Tokenization using NLTK Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "5914f389",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "showing info https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/index.xml\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[1;32mIn [14]\u001b[0m, in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mnltk\u001b[39;00m\n\u001b[0;32m      2\u001b[0m \u001b[38;5;66;03m# from urllib.request import urlopen\u001b[39;00m\n\u001b[0;32m      3\u001b[0m \u001b[38;5;66;03m# url = \"http://www.gutenberg.org/files/2554/2554.txt\"\u001b[39;00m\n\u001b[0;32m      4\u001b[0m \u001b[38;5;66;03m# raw = urlopen(url).read()\u001b[39;00m\n\u001b[0;32m      5\u001b[0m \u001b[38;5;66;03m# raw = raw.decode('utf-8')\u001b[39;00m\n\u001b[0;32m      6\u001b[0m \u001b[38;5;66;03m# tokens = nltk.word_tokenize(raw)\u001b[39;00m\n\u001b[0;32m      7\u001b[0m \u001b[38;5;66;03m# print(tokens)\u001b[39;00m\n\u001b[1;32m----> 9\u001b[0m \u001b[43mnltk\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdownload\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     10\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mnltk\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mtokenize\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m word_tokenize\n\u001b[0;32m     12\u001b[0m sentence \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mThis is an example sentence.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\nltk\\downloader.py:763\u001b[0m, in \u001b[0;36mDownloader.download\u001b[1;34m(self, info_or_id, download_dir, quiet, force, prefix, halt_on_error, raise_on_error, print_error_to)\u001b[0m\n\u001b[0;32m    761\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m download_dir \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m    762\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_download_dir \u001b[38;5;241m=\u001b[39m download_dir\n\u001b[1;32m--> 763\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_interactive_download\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    764\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[0;32m    766\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    767\u001b[0m     \u001b[38;5;66;03m# Define a helper function for displaying output:\u001b[39;00m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\nltk\\downloader.py:1113\u001b[0m, in \u001b[0;36mDownloader._interactive_download\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   1111\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m TKINTER:\n\u001b[0;32m   1112\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m-> 1113\u001b[0m         \u001b[43mDownloaderGUI\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmainloop\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1114\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m TclError:\n\u001b[0;32m   1115\u001b[0m         DownloaderShell(\u001b[38;5;28mself\u001b[39m)\u001b[38;5;241m.\u001b[39mrun()\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\nltk\\downloader.py:1932\u001b[0m, in \u001b[0;36mDownloaderGUI.mainloop\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1931\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mmainloop\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m-> 1932\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtop\u001b[38;5;241m.\u001b[39mmainloop(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\tkinter\\__init__.py:1458\u001b[0m, in \u001b[0;36mMisc.mainloop\u001b[1;34m(self, n)\u001b[0m\n\u001b[0;32m   1456\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mmainloop\u001b[39m(\u001b[38;5;28mself\u001b[39m, n\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0\u001b[39m):\n\u001b[0;32m   1457\u001b[0m     \u001b[38;5;124;03m\"\"\"Call the mainloop of Tk.\"\"\"\u001b[39;00m\n\u001b[1;32m-> 1458\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtk\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmainloop\u001b[49m\u001b[43m(\u001b[49m\u001b[43mn\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "# from urllib.request import urlopen\n",
    "# url = \"http://www.gutenberg.org/files/2554/2554.txt\"\n",
    "# raw = urlopen(url).read()\n",
    "# raw = raw.decode('utf-8')\n",
    "# tokens = nltk.word_tokenize(raw)\n",
    "# print(tokens)\n",
    "\n",
    "nltk.download()\n",
    "from nltk.tokenize import word_tokenize\n",
    "\n",
    "sentence = \"This is an example sentence.\"\n",
    "tokens = word_tokenize(sentence)\n",
    "print(tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5853d46a",
   "metadata": {},
   "source": [
    "# Stemming using NLTK library with PosterStemmer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f5699200",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading collection 'all'\n",
      "[nltk_data]    | \n",
      "[nltk_data]    | Downloading package abc to\n",
      "[nltk_data]    |     C:\\Users\\vivek\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    | Error downloading 'abc' from\n",
      "[nltk_data]    |     <https://raw.githubusercontent.com/nltk/nltk_data\n",
      "[nltk_data]    |     /gh-pages/packages/corpora/abc.zip>:   <urlopen\n",
      "[nltk_data]    |     error [WinError 10060] A connection attempt\n",
      "[nltk_data]    |     failed because the connected party did not\n",
      "[nltk_data]    |     properly respond after a period of time, or\n",
      "[nltk_data]    |     established connection failed because connected\n",
      "[nltk_data]    |     host has failed to respond>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "09d64a20",
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.stem import PorterStemmer\n",
    "from nltk.tokenize import word_tokenize\n",
    "\n",
    "def stem_file(file_path):\n",
    "    stemmer = PorterStemmer()\n",
    "    with open(file_path, \"r\") as file:\n",
    "        text = file.read()\n",
    "        words = word_tokenize(text)\n",
    "        stemmed_words = [stemmer.stem(word) for word in words]\n",
    "        print(\"Original words:\", words)\n",
    "        print(\"Stemmed words:\", stemmed_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "711b3dc1",
   "metadata": {},
   "outputs": [
    {
     "ename": "LookupError",
     "evalue": "\n**********************************************************************\n  Resource \u001b[93mpunkt\u001b[0m not found.\n  Please use the NLTK Downloader to obtain the resource:\n\n  \u001b[31m>>> import nltk\n  >>> nltk.download('punkt')\n  \u001b[0m\n  For more information see: https://www.nltk.org/data.html\n\n  Attempted to load \u001b[93mtokenizers/punkt/english.pickle\u001b[0m\n\n  Searched in:\n    - 'C:\\\\Users\\\\vivek/nltk_data'\n    - 'C:\\\\Users\\\\vivek\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python310\\\\nltk_data'\n    - 'C:\\\\Users\\\\vivek\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python310\\\\share\\\\nltk_data'\n    - 'C:\\\\Users\\\\vivek\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python310\\\\lib\\\\nltk_data'\n    - 'C:\\\\Users\\\\vivek\\\\AppData\\\\Roaming\\\\nltk_data'\n    - 'C:\\\\nltk_data'\n    - 'D:\\\\nltk_data'\n    - 'E:\\\\nltk_data'\n    - ''\n**********************************************************************\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mLookupError\u001b[0m                               Traceback (most recent call last)",
      "Input \u001b[1;32mIn [16]\u001b[0m, in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[43mstem_file\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mexample.txt\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "Input \u001b[1;32mIn [15]\u001b[0m, in \u001b[0;36mstem_file\u001b[1;34m(file_path)\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(file_path, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m file:\n\u001b[0;32m      7\u001b[0m     text \u001b[38;5;241m=\u001b[39m file\u001b[38;5;241m.\u001b[39mread()\n\u001b[1;32m----> 8\u001b[0m     words \u001b[38;5;241m=\u001b[39m \u001b[43mword_tokenize\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtext\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m      9\u001b[0m     stemmed_words \u001b[38;5;241m=\u001b[39m [stemmer\u001b[38;5;241m.\u001b[39mstem(word) \u001b[38;5;28;01mfor\u001b[39;00m word \u001b[38;5;129;01min\u001b[39;00m words]\n\u001b[0;32m     10\u001b[0m     \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mOriginal words:\u001b[39m\u001b[38;5;124m\"\u001b[39m, words)\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\nltk\\tokenize\\__init__.py:129\u001b[0m, in \u001b[0;36mword_tokenize\u001b[1;34m(text, language, preserve_line)\u001b[0m\n\u001b[0;32m    114\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mword_tokenize\u001b[39m(text, language\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124menglish\u001b[39m\u001b[38;5;124m\"\u001b[39m, preserve_line\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m):\n\u001b[0;32m    115\u001b[0m     \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m    116\u001b[0m \u001b[38;5;124;03m    Return a tokenized copy of *text*,\u001b[39;00m\n\u001b[0;32m    117\u001b[0m \u001b[38;5;124;03m    using NLTK's recommended word tokenizer\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    127\u001b[0m \u001b[38;5;124;03m    :type preserve_line: bool\u001b[39;00m\n\u001b[0;32m    128\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[1;32m--> 129\u001b[0m     sentences \u001b[38;5;241m=\u001b[39m [text] \u001b[38;5;28;01mif\u001b[39;00m preserve_line \u001b[38;5;28;01melse\u001b[39;00m \u001b[43msent_tokenize\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtext\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlanguage\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    130\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m [\n\u001b[0;32m    131\u001b[0m         token \u001b[38;5;28;01mfor\u001b[39;00m sent \u001b[38;5;129;01min\u001b[39;00m sentences \u001b[38;5;28;01mfor\u001b[39;00m token \u001b[38;5;129;01min\u001b[39;00m _treebank_word_tokenizer\u001b[38;5;241m.\u001b[39mtokenize(sent)\n\u001b[0;32m    132\u001b[0m     ]\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\nltk\\tokenize\\__init__.py:106\u001b[0m, in \u001b[0;36msent_tokenize\u001b[1;34m(text, language)\u001b[0m\n\u001b[0;32m     96\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21msent_tokenize\u001b[39m(text, language\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124menglish\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[0;32m     97\u001b[0m     \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m     98\u001b[0m \u001b[38;5;124;03m    Return a sentence-tokenized copy of *text*,\u001b[39;00m\n\u001b[0;32m     99\u001b[0m \u001b[38;5;124;03m    using NLTK's recommended sentence tokenizer\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    104\u001b[0m \u001b[38;5;124;03m    :param language: the model name in the Punkt corpus\u001b[39;00m\n\u001b[0;32m    105\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[1;32m--> 106\u001b[0m     tokenizer \u001b[38;5;241m=\u001b[39m \u001b[43mload\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43mf\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtokenizers/punkt/\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mlanguage\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[38;5;124;43m.pickle\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[0;32m    107\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m tokenizer\u001b[38;5;241m.\u001b[39mtokenize(text)\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\nltk\\data.py:750\u001b[0m, in \u001b[0;36mload\u001b[1;34m(resource_url, format, cache, verbose, logic_parser, fstruct_reader, encoding)\u001b[0m\n\u001b[0;32m    747\u001b[0m     \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m<<Loading \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mresource_url\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m>>\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m    749\u001b[0m \u001b[38;5;66;03m# Load the resource.\u001b[39;00m\n\u001b[1;32m--> 750\u001b[0m opened_resource \u001b[38;5;241m=\u001b[39m \u001b[43m_open\u001b[49m\u001b[43m(\u001b[49m\u001b[43mresource_url\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    752\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mformat\u001b[39m \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mraw\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m    753\u001b[0m     resource_val \u001b[38;5;241m=\u001b[39m opened_resource\u001b[38;5;241m.\u001b[39mread()\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\nltk\\data.py:876\u001b[0m, in \u001b[0;36m_open\u001b[1;34m(resource_url)\u001b[0m\n\u001b[0;32m    873\u001b[0m protocol, path_ \u001b[38;5;241m=\u001b[39m split_resource_url(resource_url)\n\u001b[0;32m    875\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m protocol \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mor\u001b[39;00m protocol\u001b[38;5;241m.\u001b[39mlower() \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnltk\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m--> 876\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfind\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath_\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpath\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mopen()\n\u001b[0;32m    877\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m protocol\u001b[38;5;241m.\u001b[39mlower() \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfile\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m    878\u001b[0m     \u001b[38;5;66;03m# urllib might not use mode='rb', so handle this one ourselves:\u001b[39;00m\n\u001b[0;32m    879\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m find(path_, [\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m])\u001b[38;5;241m.\u001b[39mopen()\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\nltk\\data.py:583\u001b[0m, in \u001b[0;36mfind\u001b[1;34m(resource_name, paths)\u001b[0m\n\u001b[0;32m    581\u001b[0m sep \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m*\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m*\u001b[39m \u001b[38;5;241m70\u001b[39m\n\u001b[0;32m    582\u001b[0m resource_not_found \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00msep\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00mmsg\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00msep\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m--> 583\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mLookupError\u001b[39;00m(resource_not_found)\n",
      "\u001b[1;31mLookupError\u001b[0m: \n**********************************************************************\n  Resource \u001b[93mpunkt\u001b[0m not found.\n  Please use the NLTK Downloader to obtain the resource:\n\n  \u001b[31m>>> import nltk\n  >>> nltk.download('punkt')\n  \u001b[0m\n  For more information see: https://www.nltk.org/data.html\n\n  Attempted to load \u001b[93mtokenizers/punkt/english.pickle\u001b[0m\n\n  Searched in:\n    - 'C:\\\\Users\\\\vivek/nltk_data'\n    - 'C:\\\\Users\\\\vivek\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python310\\\\nltk_data'\n    - 'C:\\\\Users\\\\vivek\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python310\\\\share\\\\nltk_data'\n    - 'C:\\\\Users\\\\vivek\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python310\\\\lib\\\\nltk_data'\n    - 'C:\\\\Users\\\\vivek\\\\AppData\\\\Roaming\\\\nltk_data'\n    - 'C:\\\\nltk_data'\n    - 'D:\\\\nltk_data'\n    - 'E:\\\\nltk_data'\n    - ''\n**********************************************************************\n"
     ]
    }
   ],
   "source": [
    "stem_file(\"example.txt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5ae3537c",
   "metadata": {},
   "source": [
    "# Lemmatization using NLTK Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f33eed9",
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk import pos_tag\n",
    "from nltk.tokenize import word_tokenize\n",
    "\n",
    "def lemmatize_file(file_path):\n",
    "    lemmatizer = WordNetLemmatizer()\n",
    "    with open(file_path, \"r\") as file:\n",
    "        text = file.read()\n",
    "        words = word_tokenize(text)\n",
    "        pos_tagged_words = pos_tag(words)\n",
    "        lemmatized_words = [lemmatizer.lemmatize(word, get_wordnet_pos(tag)) for word, tag in pos_tagged_words]\n",
    "        print(\"Original words:\", words)\n",
    "        print(\"Lemmatized words:\", lemmatized_words)\n",
    "\n",
    "def get_wordnet_pos(tag):\n",
    "    if tag.startswith('J'):\n",
    "        return wordnet.ADJ\n",
    "    elif tag.startswith('V'):\n",
    "        return wordnet.VERB\n",
    "    elif tag.startswith('N'):\n",
    "        return wordnet.NOUN\n",
    "    elif tag.startswith('R'):\n",
    "        return wordnet.ADV\n",
    "    else:\n",
    "        return wordnet.NOUN\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c1067364",
   "metadata": {},
   "outputs": [],
   "source": [
    "lemmatize_file(\"example.txt\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}