03_Data_Preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "65593177",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import string\n",
    "import re #regular expression\n",
    "import random\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cfe7cb59",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"businessCard.txt\", mode=\"r\", encoding=\"utf8\", errors=\"ignore\") as f:\n",
    "    text = f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e2eed6f6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id\ttext\ttag\n",
      "000.jpeg\t \tO\n",
      "000.jpeg\t.\tO\n",
      "000.jpeg\t040-4852\tB-PHONE\n",
      "000.jpeg\t\"8881,\"\tI-PHONE\n",
      "000.jpeg\t90309\tB-PHONE\n",
      "000.jpeg\t52549\tI-PHONE\n",
      "000.jpeg\tFi\tO\n",
      "000.jpeg\t/laurelsoverseaseducation\tO\n",
      "000.jpeg\t@:\tO\n",
      "000.jpeg\tLAURELS\tB-ORG\n",
      "000.jpeg\tOVERSEAS\tI-ORG\n",
      "000.jpeg\tEDUCATIONAL\tI-ORG\n",
      "000.jpeg\tCONSULTANCY\tI-ORG\n",
      "000.jpeg\tPVT.\tI-ORG\n",
      "000.jpeg\tLTD.\tI-ORG\n",
      "000.jpeg\tSea\tO\n",
      "000.jpeg\t|\tO\n",
      "000.jpeg\tU.K\tO\n",
      "000.jpeg\tAUSTRALIA\tO\n",
      "000.jpeg\tCANADA\tO\n",
      "000.jpeg\tIRELAND\tO\n",
      "000.jpeg\t \tO\n",
      "000.jpeg\t \tO\n",
      "000.jpeg\t \tO\n"
     ]
    }
   ],
   "source": [
    "print(text[:480])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "417317cf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'id\\ttext\\ttag\\n000.jpeg\\t \\tO\\n000.jpeg\\t.\\tO\\n000.jpeg\\t040-4852\\tB-PHONE\\n000.jpeg\\t\"8881,\"\\tI-PHONE\\n000.jpeg\\t90309\\tB-PHONE\\n000.jpeg\\t52549\\tI-PHONE\\n000.jpeg\\tFi\\tO\\n000.jpeg\\t/laurelsoverseaseducation\\tO\\n000.jpeg\\t@:\\tO\\n000.jpeg\\tLAURELS\\tB-ORG\\n000.jpeg\\tOVERSEAS\\tI-ORG\\n000.jpeg\\tEDUCATIONAL\\tI-ORG\\n000.jpeg\\tCONSULTANCY\\tI-ORG\\n000.jpeg\\tPVT.\\tI-ORG\\n000.jpeg\\tLTD.\\tI-ORG\\n000.jpeg\\tSea\\tO\\n000.jpeg\\t|\\tO\\n000.jpeg\\tU.K\\tO\\n000.jpeg\\tAUSTRALIA\\tO\\n000.jpeg\\tCANADA\\tO\\n000.jpeg\\tIRELAND\\tO\\n000.jpeg'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text[:450]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0d9180ea",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "data = list(map(lambda x: x.split(\"\\t\"), text.split(\"\\n\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6ff4e16f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['id', 'text', 'tag'],\n",
       " ['000.jpeg', ' ', 'O'],\n",
       " ['000.jpeg', '.', 'O'],\n",
       " ['000.jpeg', '040-4852', 'B-PHONE'],\n",
       " ['000.jpeg', '\"8881,\"', 'I-PHONE'],\n",
       " ['000.jpeg', '90309', 'B-PHONE'],\n",
       " ['000.jpeg', '52549', 'I-PHONE'],\n",
       " ['000.jpeg', 'Fi', 'O'],\n",
       " ['000.jpeg', '/laurelsoverseaseducation', 'O'],\n",
       " ['000.jpeg', '@:', 'O'],\n",
       " ['000.jpeg', 'LAURELS', 'B-ORG'],\n",
       " ['000.jpeg', 'OVERSEAS', 'I-ORG'],\n",
       " ['000.jpeg', 'EDUCATIONAL', 'I-ORG'],\n",
       " ['000.jpeg', 'CONSULTANCY', 'I-ORG'],\n",
       " ['000.jpeg', 'PVT.', 'I-ORG'],\n",
       " ['000.jpeg', 'LTD.', 'I-ORG'],\n",
       " ['000.jpeg', 'Sea', 'O'],\n",
       " ['000.jpeg', '|', 'O'],\n",
       " ['000.jpeg', 'U.K', 'O'],\n",
       " ['000.jpeg', 'AUSTRALIA', 'O']]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "afa54b92",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td></td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>.</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>040-4852</td>\n",
       "      <td>B-PHONE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>\"8881,\"</td>\n",
       "      <td>I-PHONE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>90309</td>\n",
       "      <td>B-PHONE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         id      text      tag\n",
       "0  000.jpeg                  O\n",
       "1  000.jpeg         .        O\n",
       "2  000.jpeg  040-4852  B-PHONE\n",
       "3  000.jpeg   \"8881,\"  I-PHONE\n",
       "4  000.jpeg     90309  B-PHONE"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(data[1:], columns=data[0])\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "52b47c48",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' \\t\\n\\r\\x0b\\x0c'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string.whitespace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8bf5b874",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string.punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7e0a3566",
   "metadata": {},
   "outputs": [],
   "source": [
    "whitespace = string.whitespace\n",
    "punctuation = '!#$%&\\'()*+:;<=>?[\\\\]^`{|}~'\n",
    "\n",
    "tableWhiteSpace = str.maketrans(\"\", \"\", whitespace)\n",
    "tablePunctuation = str.maketrans(\"\", \"\", punctuation)\n",
    "\n",
    "def clean_text(txt):\n",
    "    text = str(txt)\n",
    "    text = text.lower()\n",
    "    removewhitespace = text.translate(tableWhiteSpace)\n",
    "    removepunctuation = removewhitespace.translate(tablePunctuation)\n",
    "    \n",
    "    return str(removepunctuation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "1906db10",
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"text\"] = df[\"text\"].apply(clean_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6d9efc7d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\my\\AppData\\Local\\Temp\\ipykernel_5824\\644399253.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  dataClean.dropna(inplace=True)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>.</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>040-4852</td>\n",
       "      <td>B-PHONE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>\"8881,\"</td>\n",
       "      <td>I-PHONE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>90309</td>\n",
       "      <td>B-PHONE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>52549</td>\n",
       "      <td>I-PHONE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>fi</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>/laurelsoverseaseducation</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>@</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>laurels</td>\n",
       "      <td>B-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>overseas</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          id                       text      tag\n",
       "1   000.jpeg                          .        O\n",
       "2   000.jpeg                   040-4852  B-PHONE\n",
       "3   000.jpeg                    \"8881,\"  I-PHONE\n",
       "4   000.jpeg                      90309  B-PHONE\n",
       "5   000.jpeg                      52549  I-PHONE\n",
       "6   000.jpeg                         fi        O\n",
       "7   000.jpeg  /laurelsoverseaseducation        O\n",
       "8   000.jpeg                          @        O\n",
       "9   000.jpeg                    laurels    B-ORG\n",
       "10  000.jpeg                   overseas    I-ORG"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataClean = df.query(\"text != '' \")\n",
    "dataClean.dropna(inplace=True)\n",
    "\n",
    "dataClean.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "daf6c559",
   "metadata": {},
   "source": [
    "## Converting data to SpaCy Format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "3f4a690c",
   "metadata": {},
   "outputs": [],
   "source": [
    "group = dataClean.groupby(by=\"id\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d49d1e10",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['000.jpeg', '001.jpeg', '002.jpeg', '003.jpeg', '004.jpeg', '007.jpeg', '008.jpeg', '009.jpeg', '010.jpeg', '011.jpeg', '012.jpeg', '013.jpeg', '014.jpeg', '015.jpeg', '016.jpeg', '017.jpeg', '018.jpeg', '020.jpeg', '021.jpeg', '022.jpeg', '023.jpeg', '024.jpeg', '025.jpeg', '027.jpeg', '028.jpeg', '030.jpeg', '031.jpeg', '032.jpeg', '033.jpeg', '034.jpeg', '035.jpeg', '036.jpeg', '037.jpeg', '038.jpeg', '039.jpeg', '040.jpeg', '041.jpeg', '042.jpeg', '043.jpeg', '044.jpeg', '045.jpeg', '047.jpeg', '048.jpeg', '049.jpeg', '050.jpeg', '051.jpeg', '052.jpeg', '053.jpeg', '054.jpeg', '055.jpeg', '056.jpeg', '057.jpeg', '058.jpeg', '059.jpeg', '060.jpeg', '061.jpeg', '062.jpeg', '063.jpeg', '064.jpeg', '065.jpeg', '066.jpeg', '067.jpeg', '068.jpeg', '069.jpeg', '070.jpeg', '071.jpeg', '072.jpeg', '073.jpeg', '074.jpeg', '075.jpeg', '076.jpeg', '078.jpeg', '079.jpeg', '080.jpeg', '081.jpeg', '082.jpeg', '083.jpeg', '084.jpeg', '085.jpeg', '086.jpeg', '087.jpeg', '088.jpeg', '089.jpeg', '090.jpeg', '091.jpeg', '092.jpeg', '094.jpeg', '095.jpeg', '096.jpeg', '097.jpeg', '098.jpeg', '099.jpeg', '101.jpeg', '102.jpeg', '104.jpeg', '106.jpeg', '107.jpeg', '108.jpeg', '109.jpeg', '110.jpeg', '111.jpeg', '112.jpeg', '113.jpeg', '114.jpeg', '115.jpeg', '117.jpeg', '118.jpeg', '119.jpeg', '120.jpeg', '121.jpeg', '122.jpeg', '123.jpeg', '124.jpeg', '125.jpeg', '126.jpeg', '127.jpeg', '129.jpeg', '130.jpeg', '131.jpeg', '132.jpeg', '133.jpeg', '134.jpeg', '135.jpeg', '136.jpeg', '137.jpeg', '138.jpeg', '139.jpeg', '140.jpeg', '141.jpeg', '142.jpeg', '143.jpeg', '144.jpeg', '145.jpeg', '146.jpeg', '147.jpeg', '148.jpeg', '149.jpeg', '150.jpeg', '151.jpeg', '152.jpeg', '153.jpeg', '154.jpeg', '155.jpeg', '156.jpeg', '157.jpeg', '158.jpeg', '159.jpeg', '160.jpeg', '161.jpeg', '162.jpeg', '163.jpeg', '164.jpeg', '165.jpeg', '166.jpeg', '167.jpeg', '168.jpeg', '169.jpeg', '170.jpeg', '171.jpeg', '172.jpeg', '173.jpeg', '174.jpeg', '175.jpeg', '176.jpeg', '177.jpeg', '178.jpeg', '179.jpeg', '180.jpeg', '181.jpeg', '182.jpeg', '183.jpeg', '185.jpeg', '186.jpeg', '187.jpeg', '188.jpeg', '189.jpeg', '190.jpeg', '191.jpeg', '193.jpeg', '194.jpeg', '195.jpeg', '196.jpeg', '197.jpeg', '198.jpeg', '199.jpeg', '200.jpeg', '201.jpeg', '202.jpeg', '203.jpeg', '204.jpeg', '205.jpeg', '206.jpeg', '207.jpeg', '208.jpeg', '209.jpeg', '210.jpeg', '211.jpeg', '212.jpeg', '213.jpeg', '214.jpeg', '215.jpeg', '216.jpeg', '217.jpeg', '218.jpeg', '219.jpeg', '220.jpeg', '221.jpeg', '222.jpeg', '223.jpeg', '224.jpeg', '225.jpeg', '226.jpeg', '227.jpeg', '229.jpeg', '231.jpeg', '232.jpeg', '233.jpeg', '234.jpeg', '235.jpeg', '236.jpeg', '237.jpeg', '238.jpeg', '239.jpeg', '240.jpeg', '241.jpeg', '242.jpeg', '243.jpeg', '244.jpeg', '245.jpeg', '246.jpeg', '247.jpeg', '248.jpeg', '249.jpeg', '250.jpeg', '251.jpeg', '252.jpeg', '253.jpeg', '254.jpeg', '255.jpeg', '256.jpeg', '257.jpeg', '258.jpeg', '259.jpeg', '260.jpeg', '261.jpeg', '262.jpeg', '263.jpeg', '264.jpeg', '265.jpeg', '266.jpeg', '268.jpeg', '269.jpeg', '270.jpeg', '271.jpeg', '273.jpeg', '274.jpeg', '275.jpeg', '276.jpeg', '277.jpeg', '278.jpeg', '280.jpeg', '282.jpeg', '284.jpeg', '286.jpeg', '287.jpeg', '288.jpeg', '290.jpeg'])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "group.groups.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "65c219b7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>.</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>040-4852</td>\n",
       "      <td>B-PHONE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>\"8881,\"</td>\n",
       "      <td>I-PHONE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>90309</td>\n",
       "      <td>B-PHONE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>52549</td>\n",
       "      <td>I-PHONE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>fi</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>/laurelsoverseaseducation</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>@</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>laurels</td>\n",
       "      <td>B-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>overseas</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>educational</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>consultancy</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>pvt.</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>ltd.</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>sea</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>u.k</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>australia</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>canada</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>ireland</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>www.laurelseducation.com</td>\n",
       "      <td>B-WEB</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>info@laurelseducation.com</td>\n",
       "      <td>B-EMAIL</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          id                       text      tag\n",
       "1   000.jpeg                          .        O\n",
       "2   000.jpeg                   040-4852  B-PHONE\n",
       "3   000.jpeg                    \"8881,\"  I-PHONE\n",
       "4   000.jpeg                      90309  B-PHONE\n",
       "5   000.jpeg                      52549  I-PHONE\n",
       "6   000.jpeg                         fi        O\n",
       "7   000.jpeg  /laurelsoverseaseducation        O\n",
       "8   000.jpeg                          @        O\n",
       "9   000.jpeg                    laurels    B-ORG\n",
       "10  000.jpeg                   overseas    I-ORG\n",
       "11  000.jpeg                educational    I-ORG\n",
       "12  000.jpeg                consultancy    I-ORG\n",
       "13  000.jpeg                       pvt.    I-ORG\n",
       "14  000.jpeg                       ltd.    I-ORG\n",
       "15  000.jpeg                        sea        O\n",
       "17  000.jpeg                        u.k        O\n",
       "18  000.jpeg                  australia        O\n",
       "19  000.jpeg                     canada        O\n",
       "20  000.jpeg                    ireland        O\n",
       "27  000.jpeg   www.laurelseducation.com    B-WEB\n",
       "28  000.jpeg  info@laurelseducation.com  B-EMAIL"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "group.get_group(\"000.jpeg\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a5cad078",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([['.', 'O'],\n",
       "       ['040-4852', 'B-PHONE'],\n",
       "       ['\"8881,\"', 'I-PHONE'],\n",
       "       ['90309', 'B-PHONE'],\n",
       "       ['52549', 'I-PHONE'],\n",
       "       ['fi', 'O'],\n",
       "       ['/laurelsoverseaseducation', 'O'],\n",
       "       ['@', 'O'],\n",
       "       ['laurels', 'B-ORG'],\n",
       "       ['overseas', 'I-ORG'],\n",
       "       ['educational', 'I-ORG'],\n",
       "       ['consultancy', 'I-ORG'],\n",
       "       ['pvt.', 'I-ORG'],\n",
       "       ['ltd.', 'I-ORG'],\n",
       "       ['sea', 'O'],\n",
       "       ['u.k', 'O'],\n",
       "       ['australia', 'O'],\n",
       "       ['canada', 'O'],\n",
       "       ['ireland', 'O'],\n",
       "       ['www.laurelseducation.com', 'B-WEB'],\n",
       "       ['info@laurelseducation.com', 'B-EMAIL']], dtype=object)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grouparray = group.get_group(\"000.jpeg\")[[\"text\",\"tag\"]].values\n",
    "grouparray"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "8518fd55",
   "metadata": {},
   "outputs": [],
   "source": [
    "content = \"\"\n",
    "annotations = {\"entities\": []}\n",
    "start = 0\n",
    "end = 0\n",
    "\n",
    "for text, label in grouparray:\n",
    "    #print(text, label)\n",
    "    text = str(text)\n",
    "    stringLength = len(text) + 1\n",
    "    \n",
    "    start = end\n",
    "    end = start + stringLength\n",
    "    \n",
    "    if label != \"O\":\n",
    "        annot = (start, end-1, label)\n",
    "        annotations[\"entities\"].append(annot)\n",
    "        \n",
    "    content = content +  text + \" \""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "9d3b606a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'. 040-4852 \"8881,\" 90309 52549 fi /laurelsoverseaseducation @ laurels overseas educational consultancy pvt. ltd. sea u.k australia canada ireland www.laurelseducation.com info@laurelseducation.com '"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f5fc8fd5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'entities': [(2, 10, 'B-PHONE'),\n",
       "  (11, 18, 'I-PHONE'),\n",
       "  (19, 24, 'B-PHONE'),\n",
       "  (25, 30, 'I-PHONE'),\n",
       "  (62, 69, 'B-ORG'),\n",
       "  (70, 78, 'I-ORG'),\n",
       "  (79, 90, 'I-ORG'),\n",
       "  (91, 102, 'I-ORG'),\n",
       "  (103, 107, 'I-ORG'),\n",
       "  (108, 112, 'I-ORG'),\n",
       "  (146, 170, 'B-WEB'),\n",
       "  (171, 196, 'B-EMAIL')]}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "annotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "6ef2f776",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start: 2 end: 10\n"
     ]
    }
   ],
   "source": [
    "print(f'start: {content.find(\"040-4852\")} end: {content.find(\"040-4852\") + len(\"040-4852\")}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca782ab5",
   "metadata": {},
   "source": [
    "### Applying to all Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "b99b6b8d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['000.jpeg', '001.jpeg', '002.jpeg', '003.jpeg', '004.jpeg', '007.jpeg', '008.jpeg', '009.jpeg', '010.jpeg', '011.jpeg', '012.jpeg', '013.jpeg', '014.jpeg', '015.jpeg', '016.jpeg', '017.jpeg', '018.jpeg', '020.jpeg', '021.jpeg', '022.jpeg', '023.jpeg', '024.jpeg', '025.jpeg', '027.jpeg', '028.jpeg', '030.jpeg', '031.jpeg', '032.jpeg', '033.jpeg', '034.jpeg', '035.jpeg', '036.jpeg', '037.jpeg', '038.jpeg', '039.jpeg', '040.jpeg', '041.jpeg', '042.jpeg', '043.jpeg', '044.jpeg', '045.jpeg', '047.jpeg', '048.jpeg', '049.jpeg', '050.jpeg', '051.jpeg', '052.jpeg', '053.jpeg', '054.jpeg', '055.jpeg', '056.jpeg', '057.jpeg', '058.jpeg', '059.jpeg', '060.jpeg', '061.jpeg', '062.jpeg', '063.jpeg', '064.jpeg', '065.jpeg', '066.jpeg', '067.jpeg', '068.jpeg', '069.jpeg', '070.jpeg', '071.jpeg', '072.jpeg', '073.jpeg', '074.jpeg', '075.jpeg', '076.jpeg', '078.jpeg', '079.jpeg', '080.jpeg', '081.jpeg', '082.jpeg', '083.jpeg', '084.jpeg', '085.jpeg', '086.jpeg', '087.jpeg', '088.jpeg', '089.jpeg', '090.jpeg', '091.jpeg', '092.jpeg', '094.jpeg', '095.jpeg', '096.jpeg', '097.jpeg', '098.jpeg', '099.jpeg', '101.jpeg', '102.jpeg', '104.jpeg', '106.jpeg', '107.jpeg', '108.jpeg', '109.jpeg', '110.jpeg', '111.jpeg', '112.jpeg', '113.jpeg', '114.jpeg', '115.jpeg', '117.jpeg', '118.jpeg', '119.jpeg', '120.jpeg', '121.jpeg', '122.jpeg', '123.jpeg', '124.jpeg', '125.jpeg', '126.jpeg', '127.jpeg', '129.jpeg', '130.jpeg', '131.jpeg', '132.jpeg', '133.jpeg', '134.jpeg', '135.jpeg', '136.jpeg', '137.jpeg', '138.jpeg', '139.jpeg', '140.jpeg', '141.jpeg', '142.jpeg', '143.jpeg', '144.jpeg', '145.jpeg', '146.jpeg', '147.jpeg', '148.jpeg', '149.jpeg', '150.jpeg', '151.jpeg', '152.jpeg', '153.jpeg', '154.jpeg', '155.jpeg', '156.jpeg', '157.jpeg', '158.jpeg', '159.jpeg', '160.jpeg', '161.jpeg', '162.jpeg', '163.jpeg', '164.jpeg', '165.jpeg', '166.jpeg', '167.jpeg', '168.jpeg', '169.jpeg', '170.jpeg', '171.jpeg', '172.jpeg', '173.jpeg', '174.jpeg', '175.jpeg', '176.jpeg', '177.jpeg', '178.jpeg', '179.jpeg', '180.jpeg', '181.jpeg', '182.jpeg', '183.jpeg', '185.jpeg', '186.jpeg', '187.jpeg', '188.jpeg', '189.jpeg', '190.jpeg', '191.jpeg', '193.jpeg', '194.jpeg', '195.jpeg', '196.jpeg', '197.jpeg', '198.jpeg', '199.jpeg', '200.jpeg', '201.jpeg', '202.jpeg', '203.jpeg', '204.jpeg', '205.jpeg', '206.jpeg', '207.jpeg', '208.jpeg', '209.jpeg', '210.jpeg', '211.jpeg', '212.jpeg', '213.jpeg', '214.jpeg', '215.jpeg', '216.jpeg', '217.jpeg', '218.jpeg', '219.jpeg', '220.jpeg', '221.jpeg', '222.jpeg', '223.jpeg', '224.jpeg', '225.jpeg', '226.jpeg', '227.jpeg', '229.jpeg', '231.jpeg', '232.jpeg', '233.jpeg', '234.jpeg', '235.jpeg', '236.jpeg', '237.jpeg', '238.jpeg', '239.jpeg', '240.jpeg', '241.jpeg', '242.jpeg', '243.jpeg', '244.jpeg', '245.jpeg', '246.jpeg', '247.jpeg', '248.jpeg', '249.jpeg', '250.jpeg', '251.jpeg', '252.jpeg', '253.jpeg', '254.jpeg', '255.jpeg', '256.jpeg', '257.jpeg', '258.jpeg', '259.jpeg', '260.jpeg', '261.jpeg', '262.jpeg', '263.jpeg', '264.jpeg', '265.jpeg', '266.jpeg', '268.jpeg', '269.jpeg', '270.jpeg', '271.jpeg', '273.jpeg', '274.jpeg', '275.jpeg', '276.jpeg', '277.jpeg', '278.jpeg', '280.jpeg', '282.jpeg', '284.jpeg', '286.jpeg', '287.jpeg', '288.jpeg', '290.jpeg'])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cards = group.groups.keys()\n",
    "cards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "6fa2452a",
   "metadata": {},
   "outputs": [],
   "source": [
    "allCardsData = []\n",
    "\n",
    "for card in cards:\n",
    "    \n",
    "    cardData = []\n",
    "    grouparray = group.get_group(card)[[\"text\",\"tag\"]].values\n",
    "    content = \"\"\n",
    "    annotations = {\"entities\": []}\n",
    "    start = 0\n",
    "    end = 0\n",
    "\n",
    "    for text, label in grouparray:\n",
    "        #print(text, label)\n",
    "        text = str(text)\n",
    "        stringLength = len(text) + 1\n",
    "\n",
    "        start = end\n",
    "        end = start + stringLength\n",
    "\n",
    "        if label != \"O\":\n",
    "            annot = (start, end-1, label)\n",
    "            annotations[\"entities\"].append(annot)\n",
    "\n",
    "        content = content +  text + \" \"\n",
    "    \n",
    "    cardData = (content, annotations)\n",
    "    allCardsData.append(cardData)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "abca646e",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('. 040-4852 \"8881,\" 90309 52549 fi /laurelsoverseaseducation @ laurels overseas educational consultancy pvt. ltd. sea u.k australia canada ireland www.laurelseducation.com info@laurelseducation.com ',\n",
       "  {'entities': [(2, 10, 'B-PHONE'),\n",
       "    (11, 18, 'I-PHONE'),\n",
       "    (19, 24, 'B-PHONE'),\n",
       "    (25, 30, 'I-PHONE'),\n",
       "    (62, 69, 'B-ORG'),\n",
       "    (70, 78, 'I-ORG'),\n",
       "    (79, 90, 'I-ORG'),\n",
       "    (91, 102, 'I-ORG'),\n",
       "    (103, 107, 'I-ORG'),\n",
       "    (108, 112, 'I-ORG'),\n",
       "    (146, 170, 'B-WEB'),\n",
       "    (171, 196, 'B-EMAIL')]}),\n",
       " ('john smith marketing manager web www.psdgraphics.com phone 123-456-7890 mail email@psdgraphics.com ',\n",
       "  {'entities': [(0, 4, 'B-NAME'),\n",
       "    (5, 10, 'I-NAME'),\n",
       "    (11, 20, 'B-DES'),\n",
       "    (21, 28, 'I-DES'),\n",
       "    (33, 52, 'B-WEB'),\n",
       "    (59, 71, 'B-PHONE'),\n",
       "    (77, 98, 'B-EMAIL')]}),\n",
       " ('sau 0 98489 24441 dy \"08672,\" 224441 /enkateswapa wie ',\n",
       "  {'entities': [(6, 11, 'B-PHONE'), (12, 17, 'I-PHONE'), (37, 49, 'B-ORG')]})]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "allCardsData[:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3d6d56ed",
   "metadata": {},
   "source": [
    "### Splitting Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "22360be3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('ee highveldcoal jonathan erskine managing director bass executive \"suites,\" 1st floor virginia field \"house,\" virginia \"airport,\" durban \"north,\" 4051 south africa \"p.0,\" box \"20672,\" durban \"north,\" 4016 m.27 72539 6214 t. 27 31563 6611 f. 27 31 563 5433 / 27 86 559 2577 jono@highveldcoal.com ',\n",
       "  {'entities': [(3, 15, 'B-ORG'),\n",
       "    (16, 24, 'B-NAME'),\n",
       "    (25, 32, 'I-NAME'),\n",
       "    (33, 41, 'B-DES'),\n",
       "    (42, 50, 'I-DE'),\n",
       "    (205, 209, 'B-PHONE'),\n",
       "    (210, 215, 'I-PHONE'),\n",
       "    (216, 220, 'I-PHONE'),\n",
       "    (224, 226, 'B-PHONE'),\n",
       "    (227, 232, 'I-PHONE'),\n",
       "    (233, 237, 'I-PHONE'),\n",
       "    (273, 294, 'B-EMAIL')]}),\n",
       " ('friend unlock 34 696 900 91 pc www.trustfriendunlock.com trustfriendunlock@hotmail.com admin@trustfriendunlock.com ',\n",
       "  {'entities': [(14, 16, 'B-PHONE'),\n",
       "    (17, 20, 'I-PHONE'),\n",
       "    (21, 24, 'I-PHONE'),\n",
       "    (25, 27, 'I-PHONE'),\n",
       "    (31, 56, 'B-WEB'),\n",
       "    (57, 86, 'B-EMAIL'),\n",
       "    (87, 114, 'I-EMAIL')]})]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random.shuffle(allCardsData)\n",
    "\n",
    "allCardsData[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "51d69928",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "267"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(allCardsData)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "a5332573",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainData = allCardsData[:240]\n",
    "testData = allCardsData[240:]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "80f3af2a",
   "metadata": {},
   "source": [
    "### Save Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "935bcfef",
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(trainData, open(\"data/trainData.pickle\", mode=\"wb\"))\n",
    "pickle.dump(testData, open(\"data/testData.pickle\", mode=\"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "caf6171b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}