Precautions_Scrapping.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rRqUV9tv5Zf-"
   },
   "source": [
    "## Extracting \"Precautions\" section of the various medicines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "auVsVI-h5ZgB"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "# import these two modules bs4 for selecting HTML tags easily\n",
    "from bs4 import BeautifulSoup\n",
    "# requests module is easy to operate some people use urllib but I prefer this one because it is easy to use.\n",
    "import requests\n",
    "\n",
    "import pathlib\n",
    "from pandas import DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "DdTlea6M5ZgI"
   },
   "outputs": [],
   "source": [
    "url = 'https://www.drugs.com/npc/' \n",
    "url2 = 'https://www.drugs.com'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_SdgPD0o5ZgO"
   },
   "outputs": [],
   "source": [
    "#Requests module use to data from given url\n",
    "source=requests.get(url)\n",
    "\n",
    "# BeautifulSoup is used for getting HTML structure from requests response.\n",
    "soup=BeautifulSoup(source.text,'html')\n",
    "body = soup.body.findAll ('div',{'id': 'a2z'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZL-ko3S25ZgT"
   },
   "outputs": [],
   "source": [
    "#Extracting the links present on the web page for further exploration\n",
    "medicines = {}\n",
    "links = []\n",
    "a = 0\n",
    "for tags in body:\n",
    "    for t in tags.find_all('a', href=True):\n",
    "        links.append(t.get('href'))\n",
    "        medicines[a] = t.get_text('href')\n",
    "        a += 1\n",
    "#print(medicines)\n",
    "#print(links) \n",
    "keys = list(medicines.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WFSNvvr35ZgX"
   },
   "outputs": [],
   "source": [
    "# function for extraction of required data using HTML tags\n",
    "def item_precaution(soups_):\n",
    "    common_names = ''\n",
    "    for tags in soups_.find_all(\"div\",{\"class\": \"block yye\"}) :\n",
    "        for t in tags.find_all(\"p\"):\n",
    "            common_names  = common_names + '\\n' + t.get_text()\n",
    "            \n",
    "    if len(common_names) != 0 :\n",
    "        return(common_names)\n",
    "    else :\n",
    "        return(None)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "GxUB3xoY5Zgt",
    "outputId": "152ea4e7-a70f-4a46-d905-b68040797d67"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File does not exist\n",
      " Creating the file\n",
      "File created successfully with filename -  Precautions.csv\n",
      "No.of links are -  259\n"
     ]
    }
   ],
   "source": [
    "suc = 0\n",
    "fail = 0\n",
    "\n",
    "start = 0\n",
    "names = {}\n",
    "# First checking the file exist or not. If it exists, checking whether it is empty or not\n",
    "name = 'Precautions.csv'\n",
    "file_name = pathlib.Path(name)\n",
    "if file_name.exists ():\n",
    "    print (\"File exist.\")\n",
    "    try :\n",
    "        print(\"Reading the file now\")\n",
    "        df = pd.read_csv(name, index_col = 0)\n",
    "\n",
    "        start = (len(df) - 1)\n",
    "    except:\n",
    "        print(\"File is empty\")\n",
    "        \n",
    "    \n",
    "else:\n",
    "    print (\"File does not exist\\n\", \"Creating the file\")\n",
    "    file = open(name,\"w+\")\n",
    "    print(\"File created successfully with filename - \",name)\n",
    "    \n",
    "print(\"No.of links are - \", len(links))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8YTVmQEx5Zgy",
    "outputId": "c58f2adc-5bc2-4419-df02-b28108acb449"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Extracting data for - 5-Hydroxytryptophan\n",
      "Success in extracting data for -  5-Hydroxytryptophan\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Acai\n",
      "Success in extracting data for -  Acai\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Acerola\n",
      "No data was extracted for -  Acerola\n",
      "**************************************************\n",
      " Extracting data for - Acetyl-L-Carnitine\n",
      "Success in extracting data for -  Acetyl-L-Carnitine\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Acidophilus\n",
      "No data was extracted for -  Acidophilus\n",
      "**************************************************\n",
      " Extracting data for - Aconite\n",
      "No data was extracted for -  Aconite\n",
      "**************************************************\n",
      " Extracting data for - Adrenal Extract\n",
      "Success in extracting data for -  Adrenal Extract\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - African Mango\n",
      "Success in extracting data for -  African Mango\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Alfalfa\n",
      "Success in extracting data for -  Alfalfa\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Aloe\n",
      "Success in extracting data for -  Aloe\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Alpha-Lipoic Acid\n",
      "Success in extracting data for -  Alpha-Lipoic Acid\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Angelica\n",
      "No data was extracted for -  Angelica\n",
      "**************************************************\n",
      " Extracting data for - Anise\n",
      "No data was extracted for -  Anise\n",
      "**************************************************\n",
      " Extracting data for - Aortic Extract\n",
      "Success in extracting data for -  Aortic Extract\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Arabinoxylane\n",
      "Success in extracting data for -  Arabinoxylane\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Arginine\n",
      "Success in extracting data for -  Arginine\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Arnica\n",
      "Success in extracting data for -  Arnica\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Artichoke\n",
      "Success in extracting data for -  Artichoke\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Ashwagandha\n",
      "Success in extracting data for -  Ashwagandha\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Astragalus\n",
      "Success in extracting data for -  Astragalus\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Barberry\n",
      "No data was extracted for -  Barberry\n",
      "**************************************************\n",
      " Extracting data for - Barley Grass\n",
      "No data was extracted for -  Barley Grass\n",
      "**************************************************\n",
      " Extracting data for - Bayberry\n",
      "No data was extracted for -  Bayberry\n",
      "**************************************************\n",
      " Extracting data for - Bee Pollen\n",
      "Success in extracting data for -  Bee Pollen\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Beta Glucans\n",
      "No data was extracted for -  Beta Glucans\n",
      "**************************************************\n",
      " Extracting data for - Beta-Carotene\n",
      "Success in extracting data for -  Beta-Carotene\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Beta-Sitosterol\n",
      "Success in extracting data for -  Beta-Sitosterol\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Betaine Hydrochloride\n",
      "Success in extracting data for -  Betaine Hydrochloride\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Betel Nut\n",
      "No data was extracted for -  Betel Nut\n",
      "**************************************************\n",
      " Extracting data for - Betony\n",
      "No data was extracted for -  Betony\n",
      "**************************************************\n",
      " Extracting data for - Bilberry\n",
      "Success in extracting data for -  Bilberry\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Biotin\n",
      "Success in extracting data for -  Biotin\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Bitter Melon\n",
      "Success in extracting data for -  Bitter Melon\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Bitter Orange\n",
      "Success in extracting data for -  Bitter Orange\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Black Cohosh\n",
      "Success in extracting data for -  Black Cohosh\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Bladderwrack\n",
      "Success in extracting data for -  Bladderwrack\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Bloodroot\n",
      "No data was extracted for -  Bloodroot\n",
      "**************************************************\n",
      " Extracting data for - Boldo\n",
      "No data was extracted for -  Boldo\n",
      "**************************************************\n",
      " Extracting data for - Boneset\n",
      "No data was extracted for -  Boneset\n",
      "**************************************************\n",
      " Extracting data for - Borage\n",
      "No data was extracted for -  Borage\n",
      "**************************************************\n",
      " Extracting data for - Boron\n",
      "Success in extracting data for -  Boron\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Bovine Colostrum\n",
      "No data was extracted for -  Bovine Colostrum\n",
      "**************************************************\n",
      " Extracting data for - Brahmi\n",
      "Success in extracting data for -  Brahmi\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Branched-Chain Amino Acids\n",
      "Success in extracting data for -  Branched-Chain Amino Acids\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Brewer's Yeast\n",
      "Success in extracting data for -  Brewer's Yeast\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Bromelain\n",
      "Success in extracting data for -  Bromelain\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Buchu\n",
      "No data was extracted for -  Buchu\n",
      "**************************************************\n",
      " Extracting data for - Bugleweed\n",
      "No data was extracted for -  Bugleweed\n",
      "**************************************************\n",
      " Extracting data for - Bupleurum\n",
      "Success in extracting data for -  Bupleurum\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Burdock\n",
      "No data was extracted for -  Burdock\n",
      "**************************************************\n",
      " Extracting data for - Butcher's Broom\n",
      "No data was extracted for -  Butcher's Broom\n",
      "**************************************************\n",
      " Extracting data for - Butterbur\n",
      "Success in extracting data for -  Butterbur\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Calanolide A\n",
      "No data was extracted for -  Calanolide A\n",
      "**************************************************\n",
      " Extracting data for - Calendula\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Success in extracting data for -  Calendula\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Caprylic Acid\n",
      "Success in extracting data for -  Caprylic Acid\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Caprylidene\n",
      "Success in extracting data for -  Caprylidene\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Capsicum Peppers\n",
      "Success in extracting data for -  Capsicum Peppers\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Carnitine\n",
      "Success in extracting data for -  Carnitine\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Cascara\n",
      "Success in extracting data for -  Cascara\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Cat's Claw\n",
      "Success in extracting data for -  Cat's Claw\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Catnip\n",
      "No data was extracted for -  Catnip\n",
      "**************************************************\n",
      " Extracting data for - Centaury\n",
      "No data was extracted for -  Centaury\n",
      "**************************************************\n",
      " Extracting data for - Chamomile\n",
      "Success in extracting data for -  Chamomile\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Chaparral\n",
      "No data was extracted for -  Chaparral\n",
      "**************************************************\n",
      " Extracting data for - Chaste Tree\n",
      "Success in extracting data for -  Chaste Tree\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Chickweed\n",
      "No data was extracted for -  Chickweed\n",
      "**************************************************\n",
      " Extracting data for - Chicory\n",
      "No data was extracted for -  Chicory\n",
      "**************************************************\n",
      " Extracting data for - Chinese Cucumber\n",
      "No data was extracted for -  Chinese Cucumber\n",
      "**************************************************\n",
      " Extracting data for - Chitosan\n",
      "Success in extracting data for -  Chitosan\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Chlorophyll\n",
      "Success in extracting data for -  Chlorophyll\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Chondroitin\n",
      "Success in extracting data for -  Chondroitin\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Chromium\n",
      "Success in extracting data for -  Chromium\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Cinnamon\n",
      "Success in extracting data for -  Cinnamon\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Clove\n",
      "Success in extracting data for -  Clove\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Coleus\n",
      "Success in extracting data for -  Coleus\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Coltsfoot\n",
      "No data was extracted for -  Coltsfoot\n",
      "**************************************************\n",
      " Extracting data for - Comfrey\n",
      "No data was extracted for -  Comfrey\n",
      "**************************************************\n",
      " Extracting data for - Cordyceps\n",
      "Success in extracting data for -  Cordyceps\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Crampbark\n",
      "No data was extracted for -  Crampbark\n",
      "**************************************************\n",
      " Extracting data for - Cranberry\n",
      "Success in extracting data for -  Cranberry\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Creatine\n",
      "Success in extracting data for -  Creatine\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Cumin\n",
      "No data was extracted for -  Cumin\n",
      "**************************************************\n",
      " Extracting data for - Damiana\n",
      "No data was extracted for -  Damiana\n",
      "**************************************************\n",
      " Extracting data for - Dandelion\n",
      "Success in extracting data for -  Dandelion\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Danshen\n",
      "No data was extracted for -  Danshen\n",
      "**************************************************\n",
      " Extracting data for - Deer Velvet\n",
      "No data was extracted for -  Deer Velvet\n",
      "**************************************************\n",
      " Extracting data for - Dehydroepiandrosterone\n",
      "Success in extracting data for -  Dehydroepiandrosterone\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Devil's Claw\n",
      "Success in extracting data for -  Devil's Claw\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Dolomite\n",
      "No data was extracted for -  Dolomite\n",
      "**************************************************\n",
      " Extracting data for - Dong Quai\n",
      "Success in extracting data for -  Dong Quai\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Dragon's Blood\n",
      "No data was extracted for -  Dragon's Blood\n",
      "**************************************************\n",
      " Extracting data for - Echinacea\n",
      "Success in extracting data for -  Echinacea\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Elderberry\n",
      "Success in extracting data for -  Elderberry\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Eleutherococcus\n",
      "Success in extracting data for -  Eleutherococcus\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Emu Oil\n",
      "No data was extracted for -  Emu Oil\n",
      "**************************************************\n",
      " Extracting data for - Ephedra\n",
      "Success in extracting data for -  Ephedra\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Evening Primrose Oil\n",
      "Success in extracting data for -  Evening Primrose Oil\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Eyebright\n",
      "Success in extracting data for -  Eyebright\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - False Unicorn\n",
      "No data was extracted for -  False Unicorn\n",
      "**************************************************\n",
      " Extracting data for - Fennel\n",
      "No data was extracted for -  Fennel\n",
      "**************************************************\n",
      " Extracting data for - Fenugreek\n",
      "Success in extracting data for -  Fenugreek\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Feverfew\n",
      "Success in extracting data for -  Feverfew\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Fish Oil\n",
      "Success in extracting data for -  Fish Oil\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Flax\n",
      "Success in extracting data for -  Flax\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Fo Ti\n",
      "No data was extracted for -  Fo Ti\n",
      "**************************************************\n",
      " Extracting data for - Forskolin\n",
      "No data was extracted for -  Forskolin\n",
      "**************************************************\n",
      " Extracting data for - Frankincense, Indian\n",
      "Success in extracting data for -  Frankincense, Indian\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Fruit Acids\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Success in extracting data for -  Fruit Acids\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Fumitory\n",
      "No data was extracted for -  Fumitory\n",
      "**************************************************\n",
      " Extracting data for - Gamma Linolenic Acid (GLA)\n",
      "No data was extracted for -  Gamma Linolenic Acid (GLA)\n",
      "**************************************************\n",
      " Extracting data for - Gamma Oryzanol\n",
      "No data was extracted for -  Gamma Oryzanol\n",
      "**************************************************\n",
      " Extracting data for - Garcinia (Hydroxycitric Acid)\n",
      "Success in extracting data for -  Garcinia (Hydroxycitric Acid)\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Garlic\n",
      "Success in extracting data for -  Garlic\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Gelsemium\n",
      "No data was extracted for -  Gelsemium\n",
      "**************************************************\n",
      " Extracting data for - Gentian\n",
      "No data was extracted for -  Gentian\n",
      "**************************************************\n",
      " Extracting data for - Ginger\n",
      "Success in extracting data for -  Ginger\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Ginkgo\n",
      "Success in extracting data for -  Ginkgo\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Ginseng\n",
      "Success in extracting data for -  Ginseng\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Glucomannan\n",
      "Success in extracting data for -  Glucomannan\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Glucosamine\n",
      "Success in extracting data for -  Glucosamine\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Glutamine\n",
      "Success in extracting data for -  Glutamine\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Glutathione\n",
      "Success in extracting data for -  Glutathione\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Golden Seal\n",
      "Success in extracting data for -  Golden Seal\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Gotu Kola\n",
      "Success in extracting data for -  Gotu Kola\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Grape Seed\n",
      "Success in extracting data for -  Grape Seed\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Grapefruit\n",
      "Success in extracting data for -  Grapefruit\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Green Tea\n",
      "Success in extracting data for -  Green Tea\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Guarana\n",
      "No data was extracted for -  Guarana\n",
      "**************************************************\n",
      " Extracting data for - Guggul\n",
      "No data was extracted for -  Guggul\n",
      "**************************************************\n",
      " Extracting data for - Gymnema\n",
      "No data was extracted for -  Gymnema\n",
      "**************************************************\n",
      " Extracting data for - Hawthorn\n",
      "Success in extracting data for -  Hawthorn\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Hibiscus\n",
      "No data was extracted for -  Hibiscus\n",
      "**************************************************\n",
      " Extracting data for - Holly\n",
      "No data was extracted for -  Holly\n",
      "**************************************************\n",
      " Extracting data for - Hops\n",
      "No data was extracted for -  Hops\n",
      "**************************************************\n",
      " Extracting data for - Horehound\n",
      "No data was extracted for -  Horehound\n",
      "**************************************************\n",
      " Extracting data for - Horny Goat Weed\n",
      "Success in extracting data for -  Horny Goat Weed\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Horse Chestnut\n",
      "Success in extracting data for -  Horse Chestnut\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Horseradish\n",
      "No data was extracted for -  Horseradish\n",
      "**************************************************\n",
      " Extracting data for - Horsetail\n",
      "No data was extracted for -  Horsetail\n",
      "**************************************************\n",
      " Extracting data for - Huperzine A\n",
      "No data was extracted for -  Huperzine A\n",
      "**************************************************\n",
      " Extracting data for - Hyaluronic Acid\n",
      "Success in extracting data for -  Hyaluronic Acid\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Hyssop\n",
      "No data was extracted for -  Hyssop\n",
      "**************************************************\n",
      " Extracting data for - Inositol Hexaphosphate\n",
      "Success in extracting data for -  Inositol Hexaphosphate\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Ipecac\n",
      "No data was extracted for -  Ipecac\n",
      "**************************************************\n",
      " Extracting data for - Jojoba\n",
      "No data was extracted for -  Jojoba\n",
      "**************************************************\n",
      " Extracting data for - Juniper\n",
      "No data was extracted for -  Juniper\n",
      "**************************************************\n",
      " Extracting data for - Kalonji\n",
      "Success in extracting data for -  Kalonji\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Kaolin Hydrated Aluminum Silicate\n",
      "No data was extracted for -  Kaolin Hydrated Aluminum Silicate\n",
      "**************************************************\n",
      " Extracting data for - Kava\n",
      "Success in extracting data for -  Kava\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Kombucha\n",
      "No data was extracted for -  Kombucha\n",
      "**************************************************\n",
      " Extracting data for - Kudzu\n",
      "No data was extracted for -  Kudzu\n",
      "**************************************************\n",
      " Extracting data for - L-arginine\n",
      "No data was extracted for -  L-arginine\n",
      "**************************************************\n",
      " Extracting data for - L-Theanine\n",
      "Success in extracting data for -  L-Theanine\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - L-Tryptophan\n",
      "Success in extracting data for -  L-Tryptophan\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Lavender\n",
      "No data was extracted for -  Lavender\n",
      "**************************************************\n",
      " Extracting data for - Lecithin\n",
      "Success in extracting data for -  Lecithin\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Lemon Balm\n",
      "Success in extracting data for -  Lemon Balm\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Lemon Verbena\n",
      "No data was extracted for -  Lemon Verbena\n",
      "**************************************************\n",
      " Extracting data for - Licorice\n",
      "Success in extracting data for -  Licorice\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Linden\n",
      "No data was extracted for -  Linden\n",
      "**************************************************\n",
      " Extracting data for - Lobelia\n",
      "No data was extracted for -  Lobelia\n",
      "**************************************************\n",
      " Extracting data for - Lutein\n",
      "Success in extracting data for -  Lutein\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Lycopene\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Success in extracting data for -  Lycopene\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Lysine\n",
      "Success in extracting data for -  Lysine\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Ma Huang\n",
      "No data was extracted for -  Ma Huang\n",
      "**************************************************\n",
      " Extracting data for - Maca\n",
      "No data was extracted for -  Maca\n",
      "**************************************************\n",
      " Extracting data for - Maitake\n",
      "No data was extracted for -  Maitake\n",
      "**************************************************\n",
      " Extracting data for - Marijuana\n",
      "Success in extracting data for -  Marijuana\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Maritime Pine\n",
      "No data was extracted for -  Maritime Pine\n",
      "**************************************************\n",
      " Extracting data for - Maté\n",
      "No data was extracted for -  Maté\n",
      "**************************************************\n",
      " Extracting data for - Meadowsweet\n",
      "No data was extracted for -  Meadowsweet\n",
      "**************************************************\n",
      " Extracting data for - Melatonin\n",
      "Success in extracting data for -  Melatonin\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Methylsulfonylmethane (MSM)\n",
      "Success in extracting data for -  Methylsulfonylmethane (MSM)\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Milk Thistle\n",
      "Success in extracting data for -  Milk Thistle\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Mistletoe\n",
      "No data was extracted for -  Mistletoe\n",
      "**************************************************\n",
      " Extracting data for - Muira Puama\n",
      "No data was extracted for -  Muira Puama\n",
      "**************************************************\n",
      " Extracting data for - Mullein\n",
      "No data was extracted for -  Mullein\n",
      "**************************************************\n",
      " Extracting data for - Myrrh\n",
      "No data was extracted for -  Myrrh\n",
      "**************************************************\n",
      " Extracting data for - Neem\n",
      "No data was extracted for -  Neem\n",
      "**************************************************\n",
      " Extracting data for - Nettles\n",
      "Success in extracting data for -  Nettles\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Noni\n",
      "No data was extracted for -  Noni\n",
      "**************************************************\n",
      " Extracting data for - Oats\n",
      "No data was extracted for -  Oats\n",
      "**************************************************\n",
      " Extracting data for - Olive Leaf\n",
      "No data was extracted for -  Olive Leaf\n",
      "**************************************************\n",
      " Extracting data for - Olive Oil\n",
      "Success in extracting data for -  Olive Oil\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Onion\n",
      "No data was extracted for -  Onion\n",
      "**************************************************\n",
      " Extracting data for - Oregano\n",
      "No data was extracted for -  Oregano\n",
      "**************************************************\n",
      " Extracting data for - Pantothenic Acid\n",
      "Success in extracting data for -  Pantothenic Acid\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Papaya\n",
      "No data was extracted for -  Papaya\n",
      "**************************************************\n",
      " Extracting data for - Parsley\n",
      "No data was extracted for -  Parsley\n",
      "**************************************************\n",
      " Extracting data for - Passion Flower\n",
      "Success in extracting data for -  Passion Flower\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Pectin\n",
      "Success in extracting data for -  Pectin\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Pennyroyal\n",
      "No data was extracted for -  Pennyroyal\n",
      "**************************************************\n",
      " Extracting data for - Peppermint\n",
      "Success in extracting data for -  Peppermint\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Periwinkle\n",
      "No data was extracted for -  Periwinkle\n",
      "**************************************************\n",
      " Extracting data for - Peru Balsam\n",
      "No data was extracted for -  Peru Balsam\n",
      "**************************************************\n",
      " Extracting data for - Pineapple\n",
      "No data was extracted for -  Pineapple\n",
      "**************************************************\n",
      " Extracting data for - Plantain\n",
      "No data was extracted for -  Plantain\n",
      "**************************************************\n",
      " Extracting data for - Poison Ivy\n",
      "Success in extracting data for -  Poison Ivy\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Policosanol\n",
      "Success in extracting data for -  Policosanol\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Probiotics\n",
      "Success in extracting data for -  Probiotics\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Propolis\n",
      "No data was extracted for -  Propolis\n",
      "**************************************************\n",
      " Extracting data for - Pygeum\n",
      "Success in extracting data for -  Pygeum\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Quercetin\n",
      "Success in extracting data for -  Quercetin\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Raspberry\n",
      "Success in extracting data for -  Raspberry\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Red Clover\n",
      "Success in extracting data for -  Red Clover\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Red Yeast Rice\n",
      "Success in extracting data for -  Red Yeast Rice\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Reishi Mushroom\n",
      "No data was extracted for -  Reishi Mushroom\n",
      "**************************************************\n",
      " Extracting data for - Resveratrol\n",
      "Success in extracting data for -  Resveratrol\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Rhodiola Rosea\n",
      "Success in extracting data for -  Rhodiola Rosea\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Rose Hip\n",
      "Success in extracting data for -  Rose Hip\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Rosemary\n",
      "No data was extracted for -  Rosemary\n",
      "**************************************************\n",
      " Extracting data for - Royal Jelly\n",
      "No data was extracted for -  Royal Jelly\n",
      "**************************************************\n",
      " Extracting data for - Safflower\n",
      "Success in extracting data for -  Safflower\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Saffron\n",
      "No data was extracted for -  Saffron\n",
      "**************************************************\n",
      " Extracting data for - SAMe\n",
      "Success in extracting data for -  SAMe\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Sandalwood\n",
      "No data was extracted for -  Sandalwood\n",
      "**************************************************\n",
      " Extracting data for - Sarsaparilla\n",
      "No data was extracted for -  Sarsaparilla\n",
      "**************************************************\n",
      " Extracting data for - Sarsparilla\n",
      "No data was extracted for -  Sarsparilla\n",
      "**************************************************\n",
      " Extracting data for - Sassafras\n",
      "No data was extracted for -  Sassafras\n",
      "**************************************************\n",
      " Extracting data for - Saw Palmetto\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Success in extracting data for -  Saw Palmetto\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Schisandra\n",
      "Success in extracting data for -  Schisandra\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Scullcap\n",
      "No data was extracted for -  Scullcap\n",
      "**************************************************\n",
      " Extracting data for - Seaweed\n",
      "No data was extracted for -  Seaweed\n",
      "**************************************************\n",
      " Extracting data for - Senna\n",
      "Success in extracting data for -  Senna\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Serrapeptase\n",
      "Success in extracting data for -  Serrapeptase\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Shark Derivatives\n",
      "No data was extracted for -  Shark Derivatives\n",
      "**************************************************\n",
      " Extracting data for - Shark Liver Oil\n",
      "No data was extracted for -  Shark Liver Oil\n",
      "**************************************************\n",
      " Extracting data for - Slippery Elm\n",
      "No data was extracted for -  Slippery Elm\n",
      "**************************************************\n",
      " Extracting data for - SOD\n",
      "No data was extracted for -  SOD\n",
      "**************************************************\n",
      " Extracting data for - Soy\n",
      "Success in extracting data for -  Soy\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Spirulina\n",
      "Success in extracting data for -  Spirulina\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - St. John's Wort\n",
      "Success in extracting data for -  St. John's Wort\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Stevia\n",
      "No data was extracted for -  Stevia\n",
      "**************************************************\n",
      " Extracting data for - Sweet Wormwood\n",
      "Success in extracting data for -  Sweet Wormwood\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Taheebo\n",
      "No data was extracted for -  Taheebo\n",
      "**************************************************\n",
      " Extracting data for - Taurine\n",
      "Success in extracting data for -  Taurine\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Tea Tree Oil\n",
      "Success in extracting data for -  Tea Tree Oil\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Turmeric\n",
      "Success in extracting data for -  Turmeric\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Tylophora\n",
      "Success in extracting data for -  Tylophora\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Tyrosine\n",
      "Success in extracting data for -  Tyrosine\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Ubiquinone\n",
      "Success in extracting data for -  Ubiquinone\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Uva Ursi\n",
      "Success in extracting data for -  Uva Ursi\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Valerian\n",
      "Success in extracting data for -  Valerian\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Vanadium\n",
      "Success in extracting data for -  Vanadium\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Vervain\n",
      "No data was extracted for -  Vervain\n",
      "**************************************************\n",
      " Extracting data for - Vinpocetine\n",
      "Success in extracting data for -  Vinpocetine\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Vitamin E\n",
      "No data was extracted for -  Vitamin E\n",
      "**************************************************\n",
      " Extracting data for - Walnut\n",
      "No data was extracted for -  Walnut\n",
      "**************************************************\n",
      " Extracting data for - Wild Yam\n",
      "Success in extracting data for -  Wild Yam\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Willow Bark\n",
      "Success in extracting data for -  Willow Bark\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Wintergreen\n",
      "No data was extracted for -  Wintergreen\n",
      "**************************************************\n",
      " Extracting data for - Witch Hazel\n",
      "No data was extracted for -  Witch Hazel\n",
      "**************************************************\n",
      " Extracting data for - Yarrow\n",
      "No data was extracted for -  Yarrow\n",
      "**************************************************\n",
      " Extracting data for - Yellow Dock\n",
      "No data was extracted for -  Yellow Dock\n",
      "**************************************************\n",
      " Extracting data for - Yerba Santa\n",
      "No data was extracted for -  Yerba Santa\n",
      "**************************************************\n",
      " Extracting data for - Yogurt\n",
      "No data was extracted for -  Yogurt\n",
      "**************************************************\n",
      " Extracting data for - Yohimbe\n",
      "Success in extracting data for -  Yohimbe\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      " Extracting data for - Yucca\n",
      "No data was extracted for -  Yucca\n",
      "**************************************************\n",
      " Extracting data for - Zinc\n",
      "Success in extracting data for -  Zinc\n",
      "Copying the data from the web........\n",
      "DONE\n",
      "**************************************************\n",
      "No.of links from which data was extracted succesfully- 141\n",
      "No.of links from which data extraction was unsuccesfull- 118\n",
      "******************** PROCESS COMPLETED ********************\n"
     ]
    }
   ],
   "source": [
    "suc = 0\n",
    "fail = 0\n",
    "failed_links = []\n",
    "\n",
    "# Iterating through the links and extracting data\n",
    "for i in range(start,len(links)) :\n",
    "    print (\" Extracting data for -\", keys[i])    \n",
    "    link_url = url2 + links[i]    \n",
    "    source=requests.get(link_url)\n",
    "\n",
    "    # BeautifulSoup is used for getting HTML structure from requests response.\n",
    "    soups=BeautifulSoup(source.text,'html')\n",
    "    prec = item_precaution(soups)\n",
    "    # Avoiding errors\n",
    "    try :\n",
    "        if len(prec) != 0 :\n",
    "            print(\"Success in extracting data for - \",keys[i])\n",
    "            print(\"Copying the data from the web........\")\n",
    "            #print(prec)\n",
    "            names[keys[i]] = prec\n",
    "            print(\"DONE\")\n",
    "            print(\"*\"*50)\n",
    "            suc += 1\n",
    "\n",
    "\n",
    "        else :\n",
    "            print(\"No data was extracted for - \", keys[i])\n",
    "            names[keys[i]] = None\n",
    "            print(\"*\"*50)\n",
    "            fail += 1\n",
    "            failed_links.append(link_url)\n",
    "    except :\n",
    "        \n",
    "        print(\"No data was extracted for - \", keys[i])\n",
    "        names[keys[i]] = None\n",
    "        print(\"*\"*50)\n",
    "        fail += 1\n",
    "        failed_links.append(link_url)\n",
    "# Let's see the links from which the data extraction was a success/unsuccess\n",
    "print(\"No.of links from which data was extracted succesfully-\", suc)\n",
    "print(\"No.of links from which data extraction was unsuccesfull-\", fail)\n",
    "print(\"*\"*20,'PROCESS COMPLETED',\"*\"*20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['https://www.drugs.com/npc/acerola.html',\n",
       " 'https://www.drugs.com/npc/acidophilus.html',\n",
       " 'https://www.drugs.com/npc/aconite.html',\n",
       " 'https://www.drugs.com/npc/angelica.html',\n",
       " 'https://www.drugs.com/npc/anise.html',\n",
       " 'https://www.drugs.com/npc/barberry.html',\n",
       " 'https://www.drugs.com/npc/barley-grass.html',\n",
       " 'https://www.drugs.com/npc/bayberry.html',\n",
       " 'https://www.drugs.com/npc/beta-glucans.html',\n",
       " 'https://www.drugs.com/npc/betel-nut.html',\n",
       " 'https://www.drugs.com/npc/betony.html',\n",
       " 'https://www.drugs.com/npc/bloodroot.html',\n",
       " 'https://www.drugs.com/npc/boldo.html',\n",
       " 'https://www.drugs.com/npc/boneset.html',\n",
       " 'https://www.drugs.com/npc/borage.html',\n",
       " 'https://www.drugs.com/npc/bovine-colostrum.html',\n",
       " 'https://www.drugs.com/npc/buchu.html',\n",
       " 'https://www.drugs.com/npc/bugleweed.html',\n",
       " 'https://www.drugs.com/npc/burdock.html',\n",
       " 'https://www.drugs.com/npc/butcher-s-broom.html',\n",
       " 'https://www.drugs.com/npc/calanolide-a.html',\n",
       " 'https://www.drugs.com/npc/catnip.html',\n",
       " 'https://www.drugs.com/npc/centaury.html',\n",
       " 'https://www.drugs.com/npc/chaparral.html',\n",
       " 'https://www.drugs.com/npc/chickweed.html',\n",
       " 'https://www.drugs.com/npc/chicory.html',\n",
       " 'https://www.drugs.com/npc/chinese-cucumber.html',\n",
       " 'https://www.drugs.com/npc/coltsfoot.html',\n",
       " 'https://www.drugs.com/npc/comfrey.html',\n",
       " 'https://www.drugs.com/npc/crampbark.html',\n",
       " 'https://www.drugs.com/npc/cumin.html',\n",
       " 'https://www.drugs.com/npc/damiana.html',\n",
       " 'https://www.drugs.com/npc/danshen.html',\n",
       " 'https://www.drugs.com/npc/deer-velvet.html',\n",
       " 'https://www.drugs.com/npc/dolomite.html',\n",
       " 'https://www.drugs.com/npc/dragon-s-blood.html',\n",
       " 'https://www.drugs.com/npc/emu-oil.html',\n",
       " 'https://www.drugs.com/npc/false-unicorn.html',\n",
       " 'https://www.drugs.com/npc/fennel.html',\n",
       " 'https://www.drugs.com/npc/fo-ti.html',\n",
       " 'https://www.drugs.com/npc/forskolin.html',\n",
       " 'https://www.drugs.com/npc/fumitory.html',\n",
       " 'https://www.drugs.com/npc/gamma-linolenic-acid-gla.html',\n",
       " 'https://www.drugs.com/npc/gamma-oryzanol.html',\n",
       " 'https://www.drugs.com/npc/gelsemium.html',\n",
       " 'https://www.drugs.com/npc/gentian.html',\n",
       " 'https://www.drugs.com/npc/guarana.html',\n",
       " 'https://www.drugs.com/npc/guggul.html',\n",
       " 'https://www.drugs.com/npc/gymnema.html',\n",
       " 'https://www.drugs.com/npc/hibiscus.html',\n",
       " 'https://www.drugs.com/npc/holly.html',\n",
       " 'https://www.drugs.com/npc/hops.html',\n",
       " 'https://www.drugs.com/npc/horehound.html',\n",
       " 'https://www.drugs.com/npc/horseradish.html',\n",
       " 'https://www.drugs.com/npc/horsetail.html',\n",
       " 'https://www.drugs.com/npc/huperzine-a.html',\n",
       " 'https://www.drugs.com/npc/hyssop.html',\n",
       " 'https://www.drugs.com/npc/ipecac.html',\n",
       " 'https://www.drugs.com/npc/jojoba.html',\n",
       " 'https://www.drugs.com/npc/juniper.html',\n",
       " 'https://www.drugs.com/npc/kaolin-hydrated-aluminum-silicate.html',\n",
       " 'https://www.drugs.com/npc/kombucha.html',\n",
       " 'https://www.drugs.com/npc/kudzu.html',\n",
       " 'https://www.drugs.com/npc/l-arginine.html',\n",
       " 'https://www.drugs.com/npc/lavender.html',\n",
       " 'https://www.drugs.com/npc/lemon-verbena.html',\n",
       " 'https://www.drugs.com/npc/linden.html',\n",
       " 'https://www.drugs.com/npc/lobelia.html',\n",
       " 'https://www.drugs.com/npc/ma-huang.html',\n",
       " 'https://www.drugs.com/npc/maca.html',\n",
       " 'https://www.drugs.com/npc/maitake.html',\n",
       " 'https://www.drugs.com/npc/maritime-pine.html',\n",
       " 'https://www.drugs.com/npc/mat.html',\n",
       " 'https://www.drugs.com/npc/meadowsweet.html',\n",
       " 'https://www.drugs.com/npc/mistletoe.html',\n",
       " 'https://www.drugs.com/npc/muira-puama.html',\n",
       " 'https://www.drugs.com/npc/mullein.html',\n",
       " 'https://www.drugs.com/npc/myrrh.html',\n",
       " 'https://www.drugs.com/npc/neem.html',\n",
       " 'https://www.drugs.com/npc/noni.html',\n",
       " 'https://www.drugs.com/npc/oats.html',\n",
       " 'https://www.drugs.com/npc/olive-leaf.html',\n",
       " 'https://www.drugs.com/npc/onion.html',\n",
       " 'https://www.drugs.com/npc/oregano.html',\n",
       " 'https://www.drugs.com/npc/papaya.html',\n",
       " 'https://www.drugs.com/npc/parsley.html',\n",
       " 'https://www.drugs.com/npc/pennyroyal.html',\n",
       " 'https://www.drugs.com/npc/periwinkle.html',\n",
       " 'https://www.drugs.com/npc/peru-balsam.html',\n",
       " 'https://www.drugs.com/npc/pineapple.html',\n",
       " 'https://www.drugs.com/npc/plantain.html',\n",
       " 'https://www.drugs.com/npc/propolis.html',\n",
       " 'https://www.drugs.com/npc/reishi-mushroom.html',\n",
       " 'https://www.drugs.com/npc/rosemary.html',\n",
       " 'https://www.drugs.com/npc/royal-jelly.html',\n",
       " 'https://www.drugs.com/npc/saffron.html',\n",
       " 'https://www.drugs.com/npc/sandalwood.html',\n",
       " 'https://www.drugs.com/npc/sarsaparilla.html',\n",
       " 'https://www.drugs.com/npc/sarsparilla.html',\n",
       " 'https://www.drugs.com/npc/sassafras.html',\n",
       " 'https://www.drugs.com/npc/scullcap.html',\n",
       " 'https://www.drugs.com/npc/seaweed.html',\n",
       " 'https://www.drugs.com/npc/shark-derivatives.html',\n",
       " 'https://www.drugs.com/npc/shark-liver-oil.html',\n",
       " 'https://www.drugs.com/npc/slippery-elm.html',\n",
       " 'https://www.drugs.com/npc/sod.html',\n",
       " 'https://www.drugs.com/npc/stevia.html',\n",
       " 'https://www.drugs.com/npc/taheebo.html',\n",
       " 'https://www.drugs.com/npc/vervain.html',\n",
       " 'https://www.drugs.com/npc/vitamin-e.html',\n",
       " 'https://www.drugs.com/npc/walnut.html',\n",
       " 'https://www.drugs.com/npc/wintergreen.html',\n",
       " 'https://www.drugs.com/npc/witch-hazel.html',\n",
       " 'https://www.drugs.com/npc/yarrow.html',\n",
       " 'https://www.drugs.com/npc/yellow-dock.html',\n",
       " 'https://www.drugs.com/npc/yerba-santa.html',\n",
       " 'https://www.drugs.com/npc/yogurt.html',\n",
       " 'https://www.drugs.com/npc/yucca.html']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "failed_links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "D6kKxMiB5Zg2",
    "outputId": "6628e992-a20b-4529-975f-a4eed97e2ae4"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Medicines</th>\n",
       "      <th>Precautions</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5-Hydroxytryptophan</td>\n",
       "      <td>\\nAlways check with your doctor before you use...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Acai</td>\n",
       "      <td>\\nAlways check with your doctor before you use...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Acerola</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Acetyl-L-Carnitine</td>\n",
       "      <td>\\nAlways check with your doctor before you use...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Acidophilus</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>254</th>\n",
       "      <td>Yerba Santa</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>255</th>\n",
       "      <td>Yogurt</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>256</th>\n",
       "      <td>Yohimbe</td>\n",
       "      <td>\\nAlways check with your doctor before you use...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257</th>\n",
       "      <td>Yucca</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>258</th>\n",
       "      <td>Zinc</td>\n",
       "      <td>\\nAlways check with your doctor before you use...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>259 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               Medicines                                        Precautions\n",
       "0    5-Hydroxytryptophan  \\nAlways check with your doctor before you use...\n",
       "1                   Acai  \\nAlways check with your doctor before you use...\n",
       "2                Acerola                                               None\n",
       "3     Acetyl-L-Carnitine  \\nAlways check with your doctor before you use...\n",
       "4            Acidophilus                                               None\n",
       "..                   ...                                                ...\n",
       "254          Yerba Santa                                               None\n",
       "255               Yogurt                                               None\n",
       "256              Yohimbe  \\nAlways check with your doctor before you use...\n",
       "257                Yucca                                               None\n",
       "258                 Zinc  \\nAlways check with your doctor before you use...\n",
       "\n",
       "[259 rows x 2 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Appending the new values\n",
    "df_temp = DataFrame(list(names.items()),columns = ['Medicines','Precautions']) \n",
    "try:\n",
    "    result = pd.concat([df,df_temp], ignore_index = True)\n",
    "    result\n",
    "except :\n",
    "    result = df_temp\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "E5X2ylA55Zg5"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Medicines</th>\n",
       "      <th>Precautions</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5-Hydroxytryptophan</td>\n",
       "      <td>\\nAlways check with your doctor before you use...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Acai</td>\n",
       "      <td>\\nAlways check with your doctor before you use...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Acerola</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Acetyl-L-Carnitine</td>\n",
       "      <td>\\nAlways check with your doctor before you use...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Acidophilus</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>254</th>\n",
       "      <td>Yerba Santa</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>255</th>\n",
       "      <td>Yogurt</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>256</th>\n",
       "      <td>Yohimbe</td>\n",
       "      <td>\\nAlways check with your doctor before you use...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257</th>\n",
       "      <td>Yucca</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>258</th>\n",
       "      <td>Zinc</td>\n",
       "      <td>\\nAlways check with your doctor before you use...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>259 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               Medicines                                        Precautions\n",
       "0    5-Hydroxytryptophan  \\nAlways check with your doctor before you use...\n",
       "1                   Acai  \\nAlways check with your doctor before you use...\n",
       "2                Acerola                                               None\n",
       "3     Acetyl-L-Carnitine  \\nAlways check with your doctor before you use...\n",
       "4            Acidophilus                                               None\n",
       "..                   ...                                                ...\n",
       "254          Yerba Santa                                               None\n",
       "255               Yogurt                                               None\n",
       "256              Yohimbe  \\nAlways check with your doctor before you use...\n",
       "257                Yucca                                               None\n",
       "258                 Zinc  \\nAlways check with your doctor before you use...\n",
       "\n",
       "[259 rows x 2 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result = result.drop_duplicates()\n",
    "# writing to the file (remove \"#\" to write to the file)\n",
    "#result.to_csv(name)\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "name": "Scrapping_Common_names.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}