results.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import csv\n",
    "import xlrd\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from requests import get, session\n",
    "from bs4 import BeautifulSoup\n",
    "import matplotlib.pyplot as plt"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "data = []\n",
    "parties = {\n",
    "    'CLP': 'LNP',\n",
    "    'LP': 'LNP',\n",
    "    'NP': 'LNP',\n",
    "    'ALP': 'ALP',\n",
    "    'Australian Labor Party': 'ALP',\n",
    "    'Liberal Party': 'LNP',\n",
    "    'National Party': 'LNP',\n",
    "    'Country Liberal Party (NT)': 'LNP'\n",
    "}\n",
    "tpp = {\n",
    "    'LNP': 0,\n",
    "    'ALP': 0\n",
    "}\n",
    "\n",
    "def tpp_margin(seats_won, seats_total):\n",
    "    return round((seats_won / seats_total) * 100, 2)\n",
    "\n",
    "\n",
    "def tpp_from_csv(csv_data: str, coalition: list, alp: str):\n",
    "    lnp = 0\n",
    "    lab = 0\n",
    "    lines = csv_data.splitlines()\n",
    "    reader = csv.reader(lines, delimiter=',')\n",
    "    for row in reader:\n",
    "        if row[0] in alp:\n",
    "            lab = int(row[-2])\n",
    "        elif row[0] in coalition:\n",
    "            lnp += int(row[-2])\n",
    "    return lnp, lab\n",
    "\n",
    "\n",
    "def loop_excel_column(excel_file):\n",
    "    _tpp = dict(**tpp)\n",
    "    book = xlrd.open_workbook(excel_file)\n",
    "    sheet = book.sheet_by_index(1)\n",
    "    for _ in range(5, 153):\n",
    "        x = sheet.cell(_, 3).value\n",
    "        try:\n",
    "            _tpp[parties[x]] += 1\n",
    "        except KeyError:\n",
    "            pass\n",
    "    return _tpp\n",
    "\n",
    "\n",
    "def create_row(tpp_data: dict[str, int], seats: int, year: int, election: int):\n",
    "    return [\n",
    "        year,\n",
    "        tpp_data['LNP'],\n",
    "        tpp_data['ALP'],\n",
    "        seats,\n",
    "        tpp_margin(tpp_data['LNP'], seats),\n",
    "        tpp_margin(tpp_data['ALP'], seats),\n",
    "        election\n",
    "    ]\n",
    "\n",
    "\n",
    "def read_aec_csv(csv_url):\n",
    "    tpp_data = dict(**tpp)\n",
    "    tpp_data['LNP'], tpp_data['ALP'] = tpp_from_csv(get(csv_url).text, coalition=['LP', 'NP', 'CLP', 'LNQ', 'Liberal', 'Liberal National Party', 'The Nationals', 'Country Liberals (NT)'], alp=['ALP', 'Australian Labor Party'])\n",
    "    return tpp_data\n",
    "\n",
    "\n",
    "def read_table_from_parl_html(year: int, markup: BeautifulSoup):\n",
    "    _tpp = dict(**tpp)\n",
    "    for title in markup.find_all('h3'):\n",
    "        if str(year) in title.text:\n",
    "            table = title.findNext('table')\n",
    "    ended = False\n",
    "    for row in table.find_all('tr'):\n",
    "        if 'Seats Won' in row.text:\n",
    "            while not ended:\n",
    "                row = row.findNext('tr')\n",
    "                if 'Total' in row.text:\n",
    "                    seats = int(row.find_all('td')[-1].text.strip())\n",
    "                    ended = True\n",
    "                else:\n",
    "                    try:\n",
    "                        _tpp[parties[row.find('td').text.strip().replace('(a)', '')]] += int(row.find_all('td')[-1].text.strip())\n",
    "                    except KeyError:\n",
    "                        pass\n",
    "    return _tpp, seats\n",
    "\n",
    "\n",
    "def find_election_table(year, wa_page):\n",
    "    for election_year in wa_page.find_all('div', {'class': 'col-xs-3 elborders'}):\n",
    "        if str(year) in election_year.text:\n",
    "            return 'https://elections.uwa.edu.au/' + election_year.find('a')['href']\n",
    "\n",
    "\n",
    "def scrape_table(url, session):\n",
    "    _tpp = dict(**tpp)\n",
    "    table = BeautifulSoup(session.get(url).text, 'lxml').find('div', {'id': 'votes'}).find('tbody')\n",
    "    for row in table.find_all('tr'):\n",
    "        try:\n",
    "            _tpp[parties[row.find('td').text.strip()]] += int(row.find_all('td')[4].text.strip())\n",
    "        except KeyError:\n",
    "            pass\n",
    "    return _tpp, int(table.find_all('tr')[-1].find_all('td')[4].text.strip())"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# Elections prior to 1993\n",
    "data_report = BeautifulSoup(get('https://www.aph.gov.au/About_Parliament/Parliamentary_Departments/Parliamentary_Library/pubs/rp/rp9899/99RP08').text, 'lxml')\n",
    "\n",
    "\n",
    "rs = session()\n",
    "rs.get('https://elections.uwa.edu.au/index.lasso')\n",
    "wa_uni_data = BeautifulSoup(rs.get('https://elections.uwa.edu.au/listelections.lasso?ElectionType=2').text, 'lxml').find_all('div', {'class': 'row elrow elbordersides'})[1]"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# 1972\n",
    "_tpp, seats = scrape_table(find_election_table(1972, wa_uni_data), rs)\n",
    "data.append(create_row(\n",
    "    _tpp,\n",
    "    seats,\n",
    "    1972,\n",
    "    1\n",
    "))"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# 1974\n",
    "_tpp, seats = scrape_table(find_election_table(1974, wa_uni_data), rs)\n",
    "data.append(create_row(\n",
    "    _tpp,\n",
    "    seats,\n",
    "    1974,\n",
    "    2\n",
    "))"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# 1975\n",
    "_tpp, seats = scrape_table(find_election_table(1975, wa_uni_data), rs)\n",
    "data.append(create_row(\n",
    "    _tpp,\n",
    "    seats,\n",
    "    1975,\n",
    "    1\n",
    "))"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# 1977\n",
    "_tpp, seats = scrape_table(find_election_table(1977, wa_uni_data), rs)\n",
    "data.append(create_row(\n",
    "    _tpp,\n",
    "    seats,\n",
    "    1977,\n",
    "    2\n",
    "))"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# 1980\n",
    "_tpp, seats = scrape_table(find_election_table(1980, wa_uni_data), rs)\n",
    "data.append(create_row(\n",
    "    _tpp,\n",
    "    seats,\n",
    "    1980,\n",
    "    3\n",
    "))"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "#1983\n",
    "_tpp, seats = read_table_from_parl_html(1983, data_report)\n",
    "data.append(create_row(\n",
    "    _tpp,\n",
    "    seats,\n",
    "    1983,\n",
    "    1\n",
    "))"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "#1984\n",
    "_tpp, seats = read_table_from_parl_html(1984, data_report)\n",
    "data.append(create_row(\n",
    "    _tpp,\n",
    "    seats,\n",
    "    1984,\n",
    "    2\n",
    "))"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# 1987\n",
    "_tpp, seats = scrape_table(find_election_table(1987, wa_uni_data), rs)\n",
    "data.append(create_row(\n",
    "    _tpp,\n",
    "    seats,\n",
    "    1987,\n",
    "    3\n",
    "))"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# 1990\n",
    "_tpp, seats = scrape_table(find_election_table(1990, wa_uni_data), rs)\n",
    "data.append(create_row(\n",
    "    _tpp,\n",
    "    seats,\n",
    "    1990,\n",
    "    4\n",
    "))"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# 1993\n",
    "data.append(create_row(\n",
    "    loop_excel_column('data/1993/MEMB93.XLS'),\n",
    "    148,\n",
    "    1993,\n",
    "    5\n",
    "))"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# 1996\n",
    "data.append(create_row(\n",
    "    loop_excel_column('data/1996/MEMB96.XLS'),\n",
    "    148,\n",
    "    1996,\n",
    "    1\n",
    "))"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# 1998\n",
    "data.append(create_row(\n",
    "    loop_excel_column('data/1998/MEMB98.XLS'),\n",
    "    148,\n",
    "    1998,\n",
    "    2\n",
    "))"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# 2001\n",
    "\n",
    "tpp_2001 = dict(**tpp)\n",
    "\n",
    "workbook = xlrd.open_workbook('data/2001/Partyrep.xls')\n",
    "sheet = workbook.sheet_by_index(1)\n",
    "\n",
    "tpp_2001['LNP'] = int(sheet.cell(14,1).value + sheet.cell(14,2).value + sheet.cell(14,3).value)\n",
    "tpp_2001['ALP'] = int(sheet.cell(14,4).value)\n",
    "data.append(create_row(\n",
    "    tpp_2001,\n",
    "    150,\n",
    "    2001,\n",
    "    3\n",
    "))"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# 2004\n",
    "data.append(create_row(\n",
    "    read_aec_csv('https://results.aec.gov.au/12246/results/Downloads/HousePartyRepresentationDownload-12246.csv'),\n",
    "    150,\n",
    "    2004,\n",
    "    4\n",
    "))"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# 2007\n",
    "data.append(create_row(\n",
    "    read_aec_csv('https://results.aec.gov.au/13745/Website/Downloads/HousePartyRepresentationDownload-13745.csv'),\n",
    "    150,\n",
    "    2007,\n",
    "    1\n",
    "))"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# 2010\n",
    "data.append(create_row(\n",
    "    read_aec_csv('https://results.aec.gov.au/15508/Website/Downloads/HousePartyRepresentationDownload-15508.csv'),\n",
    "    150,\n",
    "    2010,\n",
    "    2\n",
    "))"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# 2013\n",
    "data.append(create_row(\n",
    "    read_aec_csv('https://results.aec.gov.au/17496/Website/Downloads/HousePartyRepresentationLeadingDownload-17496.csv'),\n",
    "    150,\n",
    "    2013,\n",
    "    1\n",
    "))"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# 2016\n",
    "data.append(create_row(\n",
    "    read_aec_csv('https://results.aec.gov.au/20499/Website/Downloads/HousePartyRepresentationLeadingDownload-20499.csv'),\n",
    "    150,\n",
    "    2016,\n",
    "    2\n",
    "))"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# 2016\n",
    "data.append(create_row(\n",
    "    read_aec_csv('https://results.aec.gov.au/24310/Website/Downloads/HousePartyRepresentationLeadingDownload-24310.csv'),\n",
    "    151,\n",
    "    2019,\n",
    "    3\n",
    "))"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "df = pd.DataFrame(data=data, columns=['year', 'lnp', 'alp', 'seats', 'tpp_lnp', 'tpp_alp', 'election_index'])\n",
    "df.to_csv('export.csv', index=False)\n",
    "df['year'] = pd.to_datetime(df['year'], format='%Y')\n",
    "df['election_index'] = df['election_index'].astype(str)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "fig = plt.figure()\n",
    "\n",
    "sns.lineplot(x=df['year'], y=df['tpp_lnp'], color='blue')\n",
    "sns.lineplot(x=df['year'], y=df['tpp_alp'], color='red')\n",
    "\n",
    "plt.xlabel(\"Election\")\n",
    "plt.ylabel(\"LNP v ALP seat ratio\")\n",
    "\n",
    "fig.show()"
   ],
   "outputs": [],
   "metadata": {}
  }
 ],
 "metadata": {
  "orig_nbformat": 4,
  "language_info": {
   "name": "python",
   "version": "3.9.9",
   "mimetype": "text/x-python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "pygments_lexer": "ipython3",
   "nbconvert_exporter": "python",
   "file_extension": ".py"
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.9.9 64-bit"
  },
  "interpreter": {
   "hash": "e7370f93d1d0cde622a1f8e1c04877d8463912d04d973331ad4851f04de6915a"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}