create_index.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import os\n",
    "import sys\n",
    "import nltk\n",
    "import xml.etree.cElementTree as et\n",
    "import pickle\n",
    "import base64\n",
    "import time\n",
    "from heapq import *\n",
    "import math\n",
    "import operator\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stemmer = nltk.stem.SnowballStemmer('english')\n",
    "stop_words = {}\n",
    "stop_file = open(\"stop_words.txt\", \"r\")\n",
    "words = stop_file.read()\n",
    "words = words.split(\",\")\n",
    "for word in words:\n",
    "    word = word.strip()\n",
    "    if word:\n",
    "        stop_words[word[1:-1]] = 1\n",
    "# print(stop_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pattern = re.compile(\"[^a-zA-Z0-9]\")\n",
    "cssExp = re.compile(r'{\\|(.*?)\\|}',re.DOTALL)\n",
    "linkExp = re.compile(r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+',re.DOTALL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# arguments = sys.argv\n",
    "# wikipedia_dump = arguments[0]\n",
    "wikipedia_dump = \"/home/vatsal/Documents/IIIT/Sem-3/IRE/wikipedia_extra/wiki.xml\"\n",
    "content = et.iterparse(wikipedia_dump, events=(\"start\", \"end\"))\n",
    "content = iter(content)\n",
    "# document_title = open(\"index/document_title.pickle\", \"wb\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "title_inverted_index = {}\n",
    "body_inverted_index = {}\n",
    "category_inverted_index = {}\n",
    "infobox_inverted_index = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "per_page_document = 35000\n",
    "fcount = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "document_no = 1\n",
    "title_freq = {}\n",
    "body_freq = {}\n",
    "category_freq = {}\n",
    "infobox_freq = {}\n",
    "document_title = open(\"index/titles.txt\",\"w+\")\n",
    "document_title_position = []\n",
    "document_word = {}\n",
    "start = time.time()\n",
    "store_stemmed_word = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_into_file(filename,inverted_object,flag):\n",
    "    fileptr = open(filename, \"w+\")\n",
    "    for word in sorted(inverted_object):\n",
    "        posting_list = \",\".join(inverted_object[word])\n",
    "        line = word + \"-\" + posting_list + \"\\n\"\n",
    "        fileptr.write(line)\n",
    "\n",
    "    fileptr.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_pickle_file(filename, pickleobj):\n",
    "    file = open(filename, \"wb\")\n",
    "    pickle.dump(pickleobj, file)\n",
    "    file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for event,context in content:\n",
    "    tag = re.sub(r\"{.*}\", \"\", context.tag)\n",
    "    \n",
    "    if event == \"end\":\n",
    "        \n",
    "        if tag == \"title\":\n",
    "            \n",
    "            title_text = context.text\n",
    "            position = document_title.tell()\n",
    "            document_title_position.append(position)\n",
    "            document_title.write(title_text + \"\\n\")\n",
    "            title_text = title_text.lower()\n",
    "            try:\n",
    "                words = re.split(pattern, title_text)\n",
    "                for word in words:\n",
    "                    word = word.lower()\n",
    "                    if word in store_stemmed_word:\n",
    "                        word = store_stemmed_word[word]\n",
    "                    else:\n",
    "                        stem = stemmer.stem(word)\n",
    "                        store_stemmed_word[word] = stem\n",
    "                        word = stem\n",
    "                    if len(word) <= 2:\n",
    "                        continue\n",
    "                    if word and word not in stop_words:\n",
    "                        if word not in title_freq:\n",
    "                            title_freq[word] = 1\n",
    "                        else:\n",
    "                            title_freq[word] += 1\n",
    "            except:\n",
    "                pass\n",
    "        \n",
    "        elif tag == \"text\":\n",
    "            \n",
    "            body_text = context.text\n",
    "            body_text = linkExp.sub('',str(body_text))\n",
    "            body_text = cssExp.sub('',str(body_text))\n",
    "            try:\n",
    "                category_words = re.findall(\"\\[\\[Category:(.*?)\\]\\]\", body_text);\n",
    "                if category_words != \"\":\n",
    "                    for category_word in category_words:\n",
    "                        words = re.split(pattern, category_word)\n",
    "                        for word in words:\n",
    "                            word = word.lower()\n",
    "                            if word in store_stemmed_word:\n",
    "                                word = store_stemmed_word[word]\n",
    "                            else:\n",
    "                                stem = stemmer.stem(word)\n",
    "                                store_stemmed_word[word] = stem\n",
    "                                word = stem\n",
    "                            if len(word) <= 2:\n",
    "                                continue\n",
    "                            if  word and word not in stop_words:\n",
    "                                if word not in category_freq:\n",
    "                                    category_freq[word] = 1\n",
    "                                else:\n",
    "                                    category_freq[word] += 1\n",
    "            except:\n",
    "                pass\n",
    "            \n",
    "            try:\n",
    "\n",
    "                info_words = re.findall(\"{{Infobox((.|\\n)*?)}}\", body_text)\n",
    "                if info_words != \"\":\n",
    "                    for info_word in info_words:\n",
    "                        for i_word in info_word:\n",
    "                            words = re.split(pattern, i_word)\n",
    "                            for word in words:\n",
    "                                word = word.lower()\n",
    "                                if word in store_stemmed_word:\n",
    "                                    word = store_stemmed_word[word]\n",
    "                                else:\n",
    "                                    stem = stemmer.stem(word)\n",
    "                                    store_stemmed_word[word] = stem\n",
    "                                    word = stem\n",
    "                                if len(word) <= 2:\n",
    "                                    continue\n",
    "                                if word and word not in stop_words:\n",
    "                                    if word not in infobox_freq:\n",
    "                                        infobox_freq[word] = 1\n",
    "                                    else:\n",
    "                                        infobox_freq[word] += 1\n",
    "            except:\n",
    "                pass\n",
    "                                    \n",
    "            try:\n",
    "                words = re.split(pattern, body_text)\n",
    "\n",
    "                for word in words:\n",
    "                    word = word.lower()\n",
    "                    if word in store_stemmed_word:\n",
    "                        word = store_stemmed_word[word]\n",
    "                    else:\n",
    "                        stem = stemmer.stem(word)\n",
    "                        store_stemmed_word[word] = stem\n",
    "                        word = stem\n",
    "                    if len(word) <= 2:\n",
    "                        continue\n",
    "                    if word and word not in stop_words:\n",
    "                        if word not in body_freq:\n",
    "                            body_freq[word] = 1\n",
    "                        else:\n",
    "                            body_freq[word] += 1\n",
    "            except:\n",
    "                pass\n",
    "            \n",
    "        elif tag == \"page\":\n",
    "            d_no = str(document_no)\n",
    "            for word in body_freq:\n",
    "                if word not in body_inverted_index:\n",
    "                    body_inverted_index[word]=[]\n",
    "                body_inverted_index[word].append(d_no + \":\" + str(body_freq[word]))\n",
    "            \n",
    "            body_freq.clear()\n",
    "\n",
    "            for word in title_freq:\n",
    "                if word not in title_inverted_index:\n",
    "                    title_inverted_index[word]=[]\n",
    "                title_inverted_index[word].append(d_no + \":\" + str(title_freq[word]))\n",
    "            \n",
    "            title_freq.clear()\n",
    "\n",
    "            for word in category_freq:\n",
    "                if word not in category_inverted_index:\n",
    "                    category_inverted_index[word]=[]\n",
    "                category_inverted_index[word].append(d_no + \":\" + str(category_freq[word]))\n",
    "            \n",
    "            category_freq.clear()\n",
    "            \n",
    "            for word in infobox_freq:\n",
    "                if word not in infobox_inverted_index:\n",
    "                    infobox_inverted_index[word]=[]\n",
    "                infobox_inverted_index[word].append(d_no + \":\" + str(infobox_freq[word]))\n",
    "                \n",
    "            infobox_freq.clear()\n",
    "            \n",
    "            if document_no%80000:\n",
    "                store_stemmed_word = {}\n",
    "            \n",
    "            if document_no%per_page_document==0:\n",
    "                filename = \"index/title_\" + str(fcount) + \".txt\"\n",
    "                write_into_file(filename,title_inverted_index,'t')\n",
    "                title_inverted_index.clear()\n",
    "                \n",
    "                filename = \"index/category_\" + str(fcount) + \".txt\"\n",
    "                write_into_file(filename,category_inverted_index,'c')\n",
    "                category_inverted_index.clear()\n",
    "                \n",
    "                filename = \"index/infobox_\" + str(fcount) + \".txt\"\n",
    "                write_into_file(filename,infobox_inverted_index,'i')\n",
    "                infobox_inverted_index.clear()\n",
    "                \n",
    "                filename = \"index/body_text_\" + str(fcount) + \".txt\"\n",
    "                write_into_file(filename,body_inverted_index,'b')\n",
    "                body_inverted_index.clear()\n",
    "                \n",
    "                fcount = fcount + 1\n",
    "\n",
    "            document_no += 1\n",
    "#             print(document_no)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "document_title.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = \"index/title_position.pickle\"\n",
    "write_pickle_file(filename,document_title_position)\n",
    "document_title_position.clear()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = \"index/title_\" + str(fcount) + \".txt\"\n",
    "write_into_file(filename,title_inverted_index,'t')\n",
    "title_inverted_index.clear()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = \"index/category_\" + str(fcount) + \".txt\"\n",
    "write_into_file(filename,category_inverted_index,'c')\n",
    "category_inverted_index.clear()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = \"index/infobox_\" + str(fcount) + \".txt\"\n",
    "write_into_file(filename,infobox_inverted_index,'i')\n",
    "infobox_inverted_index.clear()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = \"index/body_text_\" + str(fcount) + \".txt\"\n",
    "write_into_file(filename,body_inverted_index,'b')\n",
    "body_inverted_index.clear()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fcount = fcount + 1\n",
    "type_of_files = [\"index/title_\",\"index/category_\",\"index/infobox_\",\"index/body_text_\"]\n",
    "mapping = {}\n",
    "mapping[\"index/infobox_\"] = 'i'\n",
    "mapping[\"index/body_text_\"] = 'd'\n",
    "mapping[\"index/title_\"] = 't'\n",
    "mapping[\"index/category_\"] = 'c'\n",
    "output_files = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for f in type_of_files:\n",
    "    heap = []\n",
    "    input_files = []\n",
    "    output_file = f + \"output.txt\"\n",
    "    output_ptr = open(output_file,'w+')\n",
    "    output_files.append(output_ptr)\n",
    "    output_f_no = len(output_files) - 1\n",
    "    \n",
    "    for file_no in range(fcount):\n",
    "        fname = f + str(file_no) + \".txt\"\n",
    "        fptr = open(fname,\"r\")\n",
    "        if os.stat(fname).st_size == 0:\n",
    "            continue\n",
    "        input_files.append(fptr)\n",
    "    \n",
    "    if len(input_files) == 0:\n",
    "        break\n",
    "    \n",
    "    for file_no in range(fcount):\n",
    "        try:\n",
    "            line = input_files[file_no].readline()[:-1]\n",
    "#             print(line)\n",
    "            heap.append((line,file_no))\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "    \n",
    "    heapify(heap)\n",
    "    \n",
    "    file_no = 0\n",
    "    try:\n",
    "        while file_no < fcount:\n",
    "            line, f_no = heappop(heap)\n",
    "            word = line.split(\"-\")[0]\n",
    "            posting_list = line.split(\"-\")[1]\n",
    "            next_line = input_files[f_no].readline()[: -1]\n",
    "            if next_line!= \"\":\n",
    "                heappush(heap, (next_line, f_no))\n",
    "            else:\n",
    "                file_no = file_no + 1\n",
    "\n",
    "            while file_no < fcount:\n",
    "                try:\n",
    "                    next_line, next_f_no = heappop(heap)\n",
    "                    next_word = next_line.split(\"-\")[0]\n",
    "                    next_posting_list = next_line.split(\"-\")[1]\n",
    "                except IndexError:\n",
    "    #                 print(heap)\n",
    "    #                 print(file_no)\n",
    "    #                 print(f)\n",
    "                    break\n",
    "\n",
    "                if next_word == word:\n",
    "                    posting_list = posting_list + \",\" + next_posting_list\n",
    "                    next_next_line = input_files[next_f_no].readline()[:-1]\n",
    "                    if next_next_line:\n",
    "                        heappush(heap, (next_next_line, next_f_no))\n",
    "                    else:\n",
    "                        file_no = file_no + 1\n",
    "                else:\n",
    "                    heappush(heap, (next_line, next_f_no))\n",
    "                    break\n",
    "\n",
    "            if word not in document_word:\n",
    "                document_word[word]={}\n",
    "\n",
    "            document_word[word][mapping[f]] = output_files[output_f_no].tell()\n",
    "            final_posting_list = posting_list.split(\",\")\n",
    "\n",
    "            idf = math.log10(document_no / len(final_posting_list))\n",
    "            documents_score = {}\n",
    "            for post in final_posting_list:\n",
    "\n",
    "                doc_no = int(post.split(\":\")[0])\n",
    "                word_freq = int(post.split(\":\")[1])\n",
    "                tf = 1 + math.log10(word_freq)\n",
    "                documents_score[doc_no] = round(idf * tf, 2)\n",
    "\n",
    "            score = \"\"\n",
    "            count = 0\n",
    "            for document in documents_score.keys():\n",
    "                score = score + str(document) + \":\" + str(documents_score[document]) + \",\"\n",
    "                count = count + 1\n",
    "            score = score[:-1] + \"\\n\"\n",
    "    #             print(score)\n",
    "            output_files[output_f_no].write(score)\n",
    "    except IndexError:\n",
    "        pass\n",
    "    \n",
    "    output_files[output_f_no].close()\n",
    "    \n",
    "    for f_no in range(fcount):\n",
    "        file_ = f + str(f_no) + \".txt\"\n",
    "        os.remove(file_)\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = \"index/word_position.json\"\n",
    "# write_pickle_file(filename,document_word)\n",
    "with open(filename,\"w+\") as f:\n",
    "    json.dump(document_word,f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "end = time.time()\n",
    "print(\"Time Taken :- \",(end-start))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}