ES_Kibana_DataVisualisation_LoanPredProblem.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from time import time\n",
    "from pyelasticsearch import ElasticSearch\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_data_path = '/Users/abhilekh/Downloads/train_u6lujuX_CVtuZ9i.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_data_path = '/Users/abhilekh/Downloads/test_Y3wMUE5_7gLdaTN.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train=pd.read_csv(train_data_path);\n",
    "test=pd.read_csv(test_data_path);\n",
    "#train.head()\n",
    "#test.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "CHUNKSIZE=100\n",
    "index_name_train=\"loan_prediction_train\"\n",
    "index_name_test=\"loan_prediction_test\"\n",
    "doc_type_train=\"av-lp_train\"\n",
    "doc_type_test=\"av-lp_test\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#to get the data to elastic search\n",
    "def index_data(data_path,chunksize,index_name, doc_type):\n",
    "    f=open(data_path)\n",
    "    csvfile=pd.read_csv(f, iterator=True, chunksize=CHUNKSIZE)\n",
    "    es=ElasticSearch(\"http://localhost:9200/\") #points to the elastic search instance\n",
    "    try:        \n",
    "        es.delete_index(index_name)\n",
    "    except:\n",
    "        pass\n",
    "    es.create_index(index_name)\n",
    "    \n",
    "    for i,df in enumerate(csvfile):\n",
    "        records=df.where(pd.notnull(df),None).T.to_dict()\n",
    "        #print(records)\n",
    "        list_records=[records[it] for it in records]\n",
    "        #print(list_records)\n",
    "        try:\n",
    "            es.bulk_index(index_name,doc_type,list_records)\n",
    "        except :\n",
    "            print(\"error!, skiping chunk!\")\n",
    "            pass\n",
    "    \n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "index_data(train_data_path, CHUNKSIZE, index_name_train, doc_type_train) # Indexing train data\n",
    "\n",
    "index_data(test_data_path, CHUNKSIZE, index_name_test, doc_type_test) # Indexing test data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}