Weather_Underground_webscrapper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting selenium\n",
      "  Using cached selenium-3.141.0-py2.py3-none-any.whl (904 kB)\n",
      "Requirement already satisfied: urllib3 in e:\\anaconda\\lib\\site-packages (from selenium) (1.25.8)\n",
      "Installing collected packages: selenium\n",
      "Successfully installed selenium-3.141.0\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install selenium"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = \"C:\\Program Files (x86)\\chromedriver.exe\"\n",
    "driver = webdriver.Chrome(PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://www.wunderground.com/history/monthly/in/kolkata/VEBA/date/2016-1'\n",
    "\n",
    "driver.get(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.implicitly_wait(10)\n",
    "\n",
    "# //*[@id=\"inner-content\"]/div[2]/div[1]/div[5]/div[1]/div/lib-city-history-observation/div/div[2]/table/tbody/tr/td[4]/table\n",
    "\n",
    "that_ele = driver.find_element_by_xpath('//*[@id=\"inner-content\"]/div[2]/div[1]/div[5]/div[1]/div/lib-city-history-observation/div/div[2]/table/tbody/tr/td[4]/table')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = that_ele.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Max Avg Min\\n100 78.3 39\\n94 76.7 42\\n94 82.3 54\\n100 76.7 39\\n94 80.0 47\\n94 75.7 39\\n94 76.8 39\\n94 78.0 51\\n94 70.2 35\\n83 61.4 32\\n88 66.9 34\\n94 70.8 36\\n94 72.6 39\\n94 72.5 36\\n94 70.1 37\\n100 67.1 37\\n94 72.1 39\\n94 66.0 34\\n94 71.2 42\\n94 89.4 73\\n94 80.8 57\\n100 76.0 46\\n94 73.4 41\\n94 69.6 40\\n94 68.6 36\\n94 67.5 34\\n94 75.7 50\\n94 78.3 54\\n94 73.4 42\\n100 67.8 32\\n100 71.9 37'"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "# table = text.splitlines()\n",
    "\n",
    "driver.quit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dummy1 = float(table[1].split(' ')[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_days_in_month(year , month):\n",
    "    if(month == 1 or month == 3 or month == 5 or month == 7 or month == 8 or month == 10 or month == 12):\n",
    "        return 31\n",
    "    elif((month == 2) and (year%4 == 0) or ((year%100 == 0) and (year%400 == 0))):\n",
    "        return 29\n",
    "    elif(month == 2):\n",
    "        return 28\n",
    "    else:\n",
    "        return 30"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(get_days_in_month(2017 , 2))\n",
    "\n",
    "df = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['01-01-2016', '02-01-2016', '03-01-2016', '04-01-2016', '05-01-2016', '06-01-2016', '07-01-2016', '08-01-2016', '09-01-2016', '10-01-2016', '11-01-2016', '12-01-2016', '13-01-2016', '14-01-2016', '15-01-2016', '16-01-2016', '17-01-2016', '18-01-2016', '19-01-2016', '20-01-2016', '21-01-2016', '22-01-2016', '23-01-2016', '24-01-2016', '25-01-2016', '26-01-2016', '27-01-2016', '28-01-2016', '29-01-2016', '30-01-2016', '31-01-2016']\n"
     ]
    }
   ],
   "source": [
    "year = 2016\n",
    "month = 1\n",
    "\n",
    "month_days = get_days_in_month(year , month)\n",
    "\n",
    "date_list_month = []\n",
    "\n",
    "for i in range(1,month_days+1):\n",
    "    d = str(i)\n",
    "    m = str(month)\n",
    "    if len(d)==1:\n",
    "        d = '0' + d\n",
    "    if len(m)==1:\n",
    "        m = '0' + m\n",
    "    date = d + '-' + m + '-'+ str(year)\n",
    "    date_list_month.append(date)\n",
    "    \n",
    "print(date_list_month)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Date'] = date_list_month"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[78.3, 76.7, 82.3, 76.7, 80.0, 75.7, 76.8, 78.0, 70.2, 61.4, 66.9, 70.8, 72.6, 72.5, 70.1, 67.1, 72.1, 66.0, 71.2, 89.4, 80.8, 76.0, 73.4, 69.6, 68.6, 67.5, 75.7, 78.3, 73.4, 67.8, 71.9]\n"
     ]
    }
   ],
   "source": [
    "avg_hum = []\n",
    "\n",
    "# dummy1 = float(table[1].split(' ')[1])\n",
    "\n",
    "for i in range(1,month_days+1):\n",
    "    avg_hum.append(float(table[i].split(' ')[1]))\n",
    "    \n",
    "print(avg_hum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['RH'] = avg_hum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>RH</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01-01-2016</td>\n",
       "      <td>78.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>02-01-2016</td>\n",
       "      <td>76.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>03-01-2016</td>\n",
       "      <td>82.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>04-01-2016</td>\n",
       "      <td>76.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>05-01-2016</td>\n",
       "      <td>80.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>06-01-2016</td>\n",
       "      <td>75.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>07-01-2016</td>\n",
       "      <td>76.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>08-01-2016</td>\n",
       "      <td>78.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>09-01-2016</td>\n",
       "      <td>70.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10-01-2016</td>\n",
       "      <td>61.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>11-01-2016</td>\n",
       "      <td>66.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>12-01-2016</td>\n",
       "      <td>70.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>13-01-2016</td>\n",
       "      <td>72.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>14-01-2016</td>\n",
       "      <td>72.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>15-01-2016</td>\n",
       "      <td>70.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>16-01-2016</td>\n",
       "      <td>67.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>17-01-2016</td>\n",
       "      <td>72.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>18-01-2016</td>\n",
       "      <td>66.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>19-01-2016</td>\n",
       "      <td>71.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>20-01-2016</td>\n",
       "      <td>89.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>21-01-2016</td>\n",
       "      <td>80.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>22-01-2016</td>\n",
       "      <td>76.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>23-01-2016</td>\n",
       "      <td>73.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>24-01-2016</td>\n",
       "      <td>69.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>25-01-2016</td>\n",
       "      <td>68.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>26-01-2016</td>\n",
       "      <td>67.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>27-01-2016</td>\n",
       "      <td>75.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>28-01-2016</td>\n",
       "      <td>78.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>29-01-2016</td>\n",
       "      <td>73.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>30-01-2016</td>\n",
       "      <td>67.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>31-01-2016</td>\n",
       "      <td>71.9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Date    RH\n",
       "0   01-01-2016  78.3\n",
       "1   02-01-2016  76.7\n",
       "2   03-01-2016  82.3\n",
       "3   04-01-2016  76.7\n",
       "4   05-01-2016  80.0\n",
       "5   06-01-2016  75.7\n",
       "6   07-01-2016  76.8\n",
       "7   08-01-2016  78.0\n",
       "8   09-01-2016  70.2\n",
       "9   10-01-2016  61.4\n",
       "10  11-01-2016  66.9\n",
       "11  12-01-2016  70.8\n",
       "12  13-01-2016  72.6\n",
       "13  14-01-2016  72.5\n",
       "14  15-01-2016  70.1\n",
       "15  16-01-2016  67.1\n",
       "16  17-01-2016  72.1\n",
       "17  18-01-2016  66.0\n",
       "18  19-01-2016  71.2\n",
       "19  20-01-2016  89.4\n",
       "20  21-01-2016  80.8\n",
       "21  22-01-2016  76.0\n",
       "22  23-01-2016  73.4\n",
       "23  24-01-2016  69.6\n",
       "24  25-01-2016  68.6\n",
       "25  26-01-2016  67.5\n",
       "26  27-01-2016  75.7\n",
       "27  28-01-2016  78.3\n",
       "28  29-01-2016  73.4\n",
       "29  30-01-2016  67.8\n",
       "30  31-01-2016  71.9"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "ename": "WebDriverException",
     "evalue": "Message: chrome not reachable\n  (Session info: chrome=83.0.4103.61)\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mWebDriverException\u001b[0m                        Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-60-ad0c9a213c39>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mE:\\ANACONDA\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mclose\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    686\u001b[0m             \u001b[0mdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    687\u001b[0m         \"\"\"\n\u001b[1;32m--> 688\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mCommand\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mCLOSE\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    689\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    690\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mquit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\ANACONDA\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mexecute\u001b[1;34m(self, driver_command, params)\u001b[0m\n\u001b[0;32m    319\u001b[0m         \u001b[0mresponse\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcommand_executor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdriver_command\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    320\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 321\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merror_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcheck_response\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    322\u001b[0m             response['value'] = self._unwrap_value(\n\u001b[0;32m    323\u001b[0m                 response.get('value', None))\n",
      "\u001b[1;32mE:\\ANACONDA\\lib\\site-packages\\selenium\\webdriver\\remote\\errorhandler.py\u001b[0m in \u001b[0;36mcheck_response\u001b[1;34m(self, response)\u001b[0m\n\u001b[0;32m    240\u001b[0m                 \u001b[0malert_text\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'alert'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'text'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    241\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0malert_text\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 242\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    243\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    244\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_value_or_default\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdefault\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mWebDriverException\u001b[0m: Message: chrome not reachable\n  (Session info: chrome=83.0.4103.61)\n"
     ]
    }
   ],
   "source": [
    "driver.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "dummy_avg_rh = []\n",
    "dummy_date_list_month = []\n",
    "\n",
    "# https://www.wunderground.com/history/monthly/in/kolkata/VEBA/date/2016-1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "def just_mention_year(year):\n",
    "    for month in range(1,13):\n",
    "        url = 'https://www.wunderground.com/history/monthly/in/kolkata/VEBA/date/' + str(year) + '-' + str(month)\n",
    "        driver.get(url)\n",
    "        driver.implicitly_wait(10)\n",
    "        that_ele = driver.find_element_by_xpath('//*[@id=\"inner-content\"]/div[2]/div[1]/div[5]/div[1]/div/lib-city-history-observation/div/div[2]/table/tbody/tr/td[4]/table')        \n",
    "        driver.quit()\n",
    "        driver.implicitly_wait(1)\n",
    "        text = that_ele.text        \n",
    "        table = text.splitlines()        \n",
    "        days_in_that_month = get_days_in_month(year , month)\n",
    "        # avg_hum = []\n",
    "        # dummy1 = float(table[1].split(' ')[1])\n",
    "        for i in range(1,day_in_that_month + 1):\n",
    "            dummy_avg_rh.append(float(table[i].split(' ')[1]))\n",
    "            d = str(i)\n",
    "            m = str(month)\n",
    "            if len(d)==1:\n",
    "                d = '0' + d\n",
    "            if len(m)==1:\n",
    "                m = '0' + m\n",
    "            date = d + '-' + m + '-'+ str(year)\n",
    "            dummy_date_list_month.append(date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "just_mention_year(2016)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "year = 2016\n",
    "month = 1\n",
    "\n",
    "url = 'https://www.wunderground.com/history/monthly/in/kolkata/VEBA/date/' + str(year) + '-' + str(month)\n",
    "driver.get(url)\n",
    "driver.implicitly_wait(10)\n",
    "that_ele = driver.find_element_by_xpath('//*[@id=\"inner-content\"]/div[2]/div[1]/div[5]/div[1]/div/lib-city-history-observation/div/div[2]/table/tbody/tr/td[4]/table')        \n",
    "# driver.quit()\n",
    "# driver.implicitly_wait(1)\n",
    "text = that_ele.text      \n",
    "driver.quit()\n",
    "table = text.splitlines()        \n",
    "days_in_that_month = get_days_in_month(year , month)\n",
    "# avg_hum = []\n",
    "# dummy1 = float(table[1].split(' ')[1])\n",
    "for i in range(1,day_in_that_month + 1):\n",
    "    dummy_avg_rh.append(float(table[i].split(' ')[1]))\n",
    "    d = str(i)\n",
    "    m = str(month)\n",
    "    if len(d)==1:\n",
    "        d = '0' + d\n",
    "    if len(m)==1:\n",
    "        m = '0' + m\n",
    "    date = d + '-' + m + '-'+ str(year)\n",
    "    dummy_date_list_month.append(date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "year = 2018\n",
    "month = 8\n",
    "\n",
    "url = 'https://www.wunderground.com/history/monthly/in/kolkata/VEBA/date/' + str(year) + '-' + str(month)\n",
    "driver.get(url)\n",
    "driver.implicitly_wait(10)\n",
    "# that_ele = driver.find_element_by_xpath('//*[@id=\"inner-content\"]/div[2]/div[1]/div[5]/div[1]/div/lib-city-history-observation/div/div[2]/table/tbody/tr/td[4]/table')        \n",
    "# # driver.quit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "that_ele = driver.find_element_by_xpath('//*[@id=\"inner-content\"]/div[2]/div[1]/div[5]/div[1]/div/lib-city-history-observation/div/div[2]/table/tbody/tr/td[4]/table')        \n",
    "# driver.quit()\n",
    "# driver.implicitly_wait(1)\n",
    "text = that_ele.text      \n",
    "# driver.quit()\n",
    "table = text.splitlines()        \n",
    "days_in_that_month = get_days_in_month(year , month)\n",
    "# avg_hum = []\n",
    "# dummy1 = float(table[1].split(' ')[1])\n",
    "for i in range(1,days_in_that_month + 1):\n",
    "    dummy_avg_rh.append(float(table[i].split(' ')[1]))\n",
    "    d = str(i)\n",
    "    m = str(month)\n",
    "    if len(d)==1:\n",
    "        d = '0' + d\n",
    "    if len(m)==1:\n",
    "        m = '0' + m\n",
    "    date = d + '-' + m + '-'+ str(year)\n",
    "    dummy_date_list_month.append(date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "974"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dummy_date_list_month)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "974"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dummy_avg_rh)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Dates'] = dummy_date_list_month"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['RH'] = dummy_avg_rh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Dates</th>\n",
       "      <th>RH</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01-01-2016</td>\n",
       "      <td>78.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>02-01-2016</td>\n",
       "      <td>76.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>03-01-2016</td>\n",
       "      <td>82.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>04-01-2016</td>\n",
       "      <td>76.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>05-01-2016</td>\n",
       "      <td>80.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>969</th>\n",
       "      <td>27-08-2018</td>\n",
       "      <td>90.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>970</th>\n",
       "      <td>28-08-2018</td>\n",
       "      <td>90.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>971</th>\n",
       "      <td>29-08-2018</td>\n",
       "      <td>89.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>972</th>\n",
       "      <td>30-08-2018</td>\n",
       "      <td>85.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>973</th>\n",
       "      <td>31-08-2018</td>\n",
       "      <td>91.6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>974 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          Dates    RH\n",
       "0    01-01-2016  78.3\n",
       "1    02-01-2016  76.7\n",
       "2    03-01-2016  82.3\n",
       "3    04-01-2016  76.7\n",
       "4    05-01-2016  80.0\n",
       "..          ...   ...\n",
       "969  27-08-2018  90.5\n",
       "970  28-08-2018  90.2\n",
       "971  29-08-2018  89.5\n",
       "972  30-08-2018  85.2\n",
       "973  31-08-2018  91.6\n",
       "\n",
       "[974 rows x 2 columns]"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv (r'E:\\RH-2016_2018-8_dataframe.csv' , index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}