model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Below is the model for Fake News detection over the Buzzfeed-Webis Fake News Corpus 2016. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "np.random.seed(1337)\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV # Tools for splitting data, tuning hyperparameters\n",
    "from sklearn.linear_model import LogisticRegressionCV # Logreg model\n",
    "from sklearn.metrics import classification_report, accuracy_score, confusion_matrix # Evaluation tools\n",
    "from nltk.tokenize import word_tokenize # Tokenizer\n",
    "\n",
    "\n",
    "import xml.etree.ElementTree as ET\n",
    "import os\n",
    "from gensim.models import Word2Vec, KeyedVectors\n",
    "\n",
    "# CNN - Second Implementation (Medium post)\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Dense, Flatten, Embedding, Input, Conv1D, MaxPooling1D, Dropout, LSTM\n",
    "from keras.utils import to_categorical\n",
    "from keras import backend as K\n",
    "from keras.regularizers import l2\n",
    "from keras.wrappers.scikit_learn import KerasClassifier # Wrapper to use Keras model in sklearn\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The first step is to download the Buzzfeed-Webis corpus, which is provided in the form of XML files. read_files will read each file and parse the XML tree to retrieve a tuple of the body of the text ('mainText') and the veracity label ('veracity')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "possibilities = ['mixture of true and false', 'mostly false', 'mostly true']\n",
    "# possibilities = ['mixture of true and false', 'mostly false', 'no factual content', 'mostly true']\n",
    "\n",
    "\n",
    "def read_files(cols, orientation=\"all\"):\n",
    "    \"\"\"\n",
    "    For each xml file return a matrix of values asked for\n",
    "    \"\"\"\n",
    "    path = 'data/train/'\n",
    "    for filename in os.listdir(path):\n",
    "        data_row = []\n",
    "        if not filename.endswith('.xml'): continue\n",
    "        xmlfile = os.path.join(path, filename)\n",
    "        tree = ET.parse(xmlfile)\n",
    "\n",
    "# FOR TESTING WHOLE DATASET, use below:        \n",
    "#         if not tree.find(\"mainText\").text: continue\n",
    "#         if orientation != \"all\" and tree.find(\"orientation\").text != orientation:\n",
    "#             continue\n",
    "        \n",
    "#COMMENT IN THE FOLLOWING two if statements to test the partisan-only dataset\n",
    "        if not tree.find(\"mainText\").text or tree.find(\"veracity\").text == \"no factual content\": continue\n",
    "        if orientation == \"all\" and tree.find(\"orientation\").text == 'mainstream':\n",
    "            continue    \n",
    "        if cols == \"mainText\":\n",
    "            if tree.find(\"mainText\").text:\n",
    "                yield tree.find(\"mainText\").text\n",
    "            else:\n",
    "                continue\n",
    "        elif cols == \"veracity\":\n",
    "            v = possibilities.index(tree.find(\"veracity\").text)\n",
    "            yield v\n",
    "        elif cols == \"both\":\n",
    "            if tree.find(\"mainText\").text:\n",
    "                v = possibilities.index(tree.find(\"veracity\").text)\n",
    "                yield tree.find(\"mainText\").text, v\n",
    "            else:\n",
    "                continue\n",
    "        else:\n",
    "            for col in cols:\n",
    "                try:\n",
    "                    data_row.append(float(tree.find(col).text))\n",
    "                except:\n",
    "                    data_row.append(0.0)\n",
    "            yield data_row"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We call this function to get a list of the main text of each article ('documents') as well as a matching list of the labels ('predictions')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "730 201 82 447\n"
     ]
    }
   ],
   "source": [
    "def feature_matrix(cols):\n",
    "    data = []\n",
    "    for row in read_files(cols):\n",
    "        data.append(np.array(row))\n",
    "    return np.array(data)\n",
    "\n",
    "def get_document_text():\n",
    "    data = []\n",
    "    for row in read_files(\"mainText\"):\n",
    "        if not row:\n",
    "            continue\n",
    "        else:\n",
    "            data.append(row)\n",
    "    return data\n",
    "\n",
    "def get_veracity():\n",
    "    data = []\n",
    "    for row in read_files(\"veracity\"):\n",
    "        data.append(row)\n",
    "    return data\n",
    "\n",
    "def get_document_text_and_veracity():\n",
    "    docs, preds = [], []\n",
    "    for row in read_files(\"both\"):\n",
    "        if not row[0]:\n",
    "            continue\n",
    "        else:\n",
    "            docs.append(row[0])\n",
    "            preds.append(row[1])\n",
    "    return docs, preds\n",
    "\n",
    "documents, predictions = get_document_text_and_veracity()\n",
    "\n",
    "print(len(documents),predictions.count(0),predictions.count(1),predictions.count(2))\n",
    "# maximum = 0\n",
    "# minimum = 1000\n",
    "# for doc in documents:\n",
    "#     l = word_tokenize(doc)\n",
    "#     if len(l) > maximum:\n",
    "#         maximum = len(l)\n",
    "#     if len(l) < minimum:\n",
    "#         minimum = len(l)    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we load the Google News pre-trained word embeddings for use in our model. These embeddings are trained using a combination of CBOW and skip-grams over a corpus of over 100 billion words from Google News.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "file = 'data/GoogleNews-vectors-negative300.bin'\n",
    "embeddings = KeyedVectors.load_word2vec_format(file, binary=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To represent entire articles using the Google News word embeddings, we replace each string with it's matching embedding and then taken the elementwise mean, max or min of the entire document. This takes a document of N words from being N separate vectors to being a single 1D vector (300 by 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def avg_docvec(docText,embeddings):\n",
    "    \"\"\"\n",
    "    This function converts the text of a document (input as a string) to word embeddings, then\n",
    "    takes the elementwise average of the embeddings to return a single vector.\n",
    "    \"\"\"\n",
    "    docVec = np.zeros(300) # Initialize array for the document\n",
    "    tokens = word_tokenize(docText) # Creates a list of word tokens (e.g. \"Test words\" -> ['Test', 'words'])\n",
    "    denominator = 0.0 # To take the average, will only count tokens for which we have embeddings in the total  \n",
    "    for token in tokens:\n",
    "        try:\n",
    "            v = embeddings[token]\n",
    "            np.add(docVec,v,out=docVec)\n",
    "            denominator += 1.0\n",
    "        except: # Ignore tokens that aren't in the Google News embeddings\n",
    "            continue\n",
    "    np.divide(docVec,denominator,out=docVec) \n",
    "    return docVec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def max_docvec(docText,embeddings):\n",
    "    \"\"\"\n",
    "    Converts the text of a document (input as a string) to word embeddings, then takes the elementwise\n",
    "    max of the embeddings to return a single vector of the maximum elements.\n",
    "    \"\"\"\n",
    "    docVec = 0\n",
    "    tokens = word_tokenize(docText) # Creates a list of word tokens (e.g. \"Test words\" -> ['Test', 'words'])\n",
    "    startIndex = 0\n",
    "    for i in range(len(tokens)): # Initialize the doc vec as the first token that is in the embeddings\n",
    "        try:\n",
    "            v = embeddings[tokens[i]]\n",
    "            docVec = v\n",
    "            startIndex = i\n",
    "            break\n",
    "        except:\n",
    "            continue\n",
    "    \n",
    "    for token in tokens[startIndex:]:\n",
    "        try:\n",
    "            v = embeddings[token]\n",
    "            np.max(docVec,v,out=docVec)\n",
    "        except: # Ignore tokens that aren't in the Google News embeddings\n",
    "            continue\n",
    "    return docVec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def min_docvec(docText,embeddings):\n",
    "    \"\"\"\n",
    "    Converts the text of a document (input as a string) to word embeddings, then takes the elementwise\n",
    "    min of the embeddings to return a single vector of the minimum elements.\n",
    "    \"\"\"\n",
    "    docVec = 0\n",
    "    tokens = word_tokenize(docText) # Creates a list of word tokens (e.g. \"Test words\" -> ['Test', 'words'])\n",
    "    startIndex = 0\n",
    "    for i in range(len(tokens)): # Initialize the doc vec as the first token that is in the embeddings\n",
    "        try:\n",
    "            v = embeddings[tokens[i]]\n",
    "            docVec = v\n",
    "            startIndex = i\n",
    "            break\n",
    "        except:\n",
    "            continue\n",
    "    for token in tokens[startIndex:]: # Loop over words in the article, starting at first valid word\n",
    "        try:\n",
    "            v = embeddings[token]\n",
    "            np.min(docVec,v,out=docVec) # Only keep min elements\n",
    "        except: # Ignore tokens that aren't in the Google News embeddings\n",
    "            continue\n",
    "    return docVec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def docs_to_matrix(documents,embeddings,method='avg'):\n",
    "    \"\"\"\n",
    "    Takes a list of document text strings and returns a matrix of document embeddings.\n",
    "    The method specifies how the word vectors are combined for the document: average is \n",
    "    element-wise average, min is element-wise min and max is element-wise max. \n",
    "    \"\"\"\n",
    "    matrix = []\n",
    "    count = 0\n",
    "    for i in range(len(documents)):\n",
    "        vector = 0\n",
    "        if method.lower() == 'avg':\n",
    "            vector = avg_docvec(documents[i],embeddings)\n",
    "        elif method.lower() == 'min':\n",
    "            vector = min_docvec(documents[i],embeddings)\n",
    "        elif method.lower() == 'max':\n",
    "            vector = max_docvec(documents[i],embeddings)\n",
    "        else:\n",
    "            print(\"Please enter method argument as min, max or avg\")\n",
    "            return\n",
    "        if i == 0:\n",
    "            matrix = vector\n",
    "        else:\n",
    "            matrix = np.column_stack((matrix,vector)) # Concat all vectors into a matrix of order (300,N of docs)\n",
    "            count += 1\n",
    "    matrix = matrix.reshape((len(documents),300)) # For sklearn, reshape the matrix into order (N of docs,300), so rows = docs\n",
    "    return matrix\n",
    "\n",
    "articles_matrix = docs_to_matrix(documents,embeddings,'avg')     "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we make the training and test datasets using the article representations and predictions. Then we create a logistic regression object, fit the model on the training data and check it on the test info.\n",
    "\n",
    "#### Notes on testing: \n",
    "* A majority classifier gets outcome Precision = 0.62, Recall = 0.78, F = 0.69. Unregularized tf-idf acts as a majority classifier.\n",
    "* Using regularized tf-idf logreg (C = 100), Precision/Recall/F =  0.82, 0.80, 0.73. This is no longer a majority classifier. \n",
    "* Representing articles as elementwise average of the word embeddings does not change the model from being a majority classifier, even when using extreme regularization (C = .0001). This applies for min and max document vectors as well. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Use the TF-IDF representation of each article as a feature matrix\n",
    "sklearn_tfidf = TfidfVectorizer(norm='l2',min_df=0, use_idf=True, smooth_idf=False, sublinear_tf=True, tokenizer=word_tokenize)\n",
    "articles_matrix = sklearn_tfidf.fit_transform(documents)\n",
    "X_train, X_test, y_train, y_test = train_test_split(articles_matrix, predictions, test_size = .3, random_state=25)\n",
    "\n",
    "# Use the combination of pre-trained word embeddings in each article as a feature matrix\n",
    "# X_train, X_test, y_train, y_test = train_test_split(articles_matrix, predictions, test_size = .3, random_state=25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(511, 20658)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape\n",
    "# articles_matrix.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/Tim/anaconda/lib/python3.6/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Cs': [1e-05, 0.0001, 0.001, 0.01, 0.1, 0.2, 0.5, 0.8, 1, 2, 5, 10, 100, 1000], 'class_weight': None, 'cv': None, 'dual': False, 'fit_intercept': True, 'intercept_scaling': 1.0, 'max_iter': 100, 'multi_class': 'ovr', 'n_jobs': 1, 'penalty': 'l2', 'random_state': None, 'refit': True, 'scoring': 'f1', 'solver': 'lbfgs', 'tol': 0.0001, 'verbose': 0}\n"
     ]
    }
   ],
   "source": [
    "# Logistic Regression\n",
    "logreg = LogisticRegressionCV(penalty='l2', scoring=\"f1\",Cs=[.00001,.0001,.001,.01,.1,.2,.5,.8,1,2,5,10,100,1000])\n",
    "logreg.fit(X_train,y_train)\n",
    "y_pred = logreg.predict(X_test)\n",
    "print(logreg.get_params())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Model Evaluation: \n",
    "Here we evaluate the effectiveness of our model using a confusion matrix, precision, recall and F1-score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[  3   0   1  53]\n",
      " [  2   4   0  24]\n",
      " [  0   2   0  15]\n",
      " [ 14   5   4 355]]\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.16      0.05      0.08        57\n",
      "          1       0.36      0.13      0.20        30\n",
      "          2       0.00      0.00      0.00        17\n",
      "          3       0.79      0.94      0.86       378\n",
      "\n",
      "avg / total       0.66      0.75      0.70       482\n",
      "\n",
      "0.751037344398\n"
     ]
    }
   ],
   "source": [
    "conf_matrix = confusion_matrix(y_test, y_pred)\n",
    "print(conf_matrix)\n",
    "print(classification_report(y_test, y_pred))\n",
    "print(accuracy_score(y_test,y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# CNN Implementation based off Keras models\n",
    "# Best for this seed? len=800,dim=300,dropout=0.5,filters=300,k=5 \n",
    "\n",
    "# Define hyperparameters\n",
    "MAX_SEQUENCE_LENGTH = 700\n",
    "TEST_SPLIT = 0.2\n",
    "VALIDATION_SPLIT = 0.2\n",
    "EMBEDDING_DIM = 300 # Google News embeddings are 300 dimensional \n",
    "DROPOUT = 0.5 # Dropout strength \n",
    "FILTERS = 300 # Number of filters in the convolutional layers\n",
    "k = 6 # Sliding k window size for convolutional layers\n",
    "\n",
    "# Prepare tokenizer\n",
    "t = Tokenizer()\n",
    "t.fit_on_texts(documents)\n",
    "vocab_size = len(t.word_index) + 1\n",
    "\n",
    "# integer encode the documents\n",
    "encoded_docs = t.texts_to_sequences(documents)\n",
    "\n",
    "# pad our doc sequences to a max length of MAX_SEQUENCE_LENGTH words\n",
    "data = pad_sequences(encoded_docs, maxlen=MAX_SEQUENCE_LENGTH, padding='post')\n",
    "\n",
    "# Re-using the list of integer labels generated earlier, make a binary class matrix\n",
    "labels = to_categorical(np.asarray(predictions))\n",
    "possibilities = ['mixture of true and false', 'mostly false', 'no factual content', 'mostly true']\n",
    "# Split into train and test sets - I try 2 versions, one from Keras and one using sklearn functions\n",
    "\n",
    "# Split the full dataset into data used for training and the final test stage\n",
    "x_training_set, x_final_test, y_training_set, y_final_test = train_test_split(data, labels, test_size = TEST_SPLIT, random_state=25) \n",
    "\n",
    "# Secondary split of training data into test and training\n",
    "x_train, x_dev, y_train, y_dev = train_test_split(x_training_set, y_training_set, test_size = VALIDATION_SPLIT, random_state=17)\n",
    "\n",
    "# create a weight matrix for words in training docs\n",
    "\n",
    "embedding_weights = np.zeros((vocab_size, 300))\n",
    "for word, i in t.word_index.items():\n",
    "    embedding_vector = None\n",
    "    try:\n",
    "        embedding_vector = embeddings[word] # Get the vector for a given word\n",
    "    except:\n",
    "        embedding_vector = None\n",
    "    if embedding_vector is not None:\n",
    "        embedding_weights[i] = embedding_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3, 3, 3, 0, 3, 3, 3, 3, 3, 0, 3, 3, 3, 3, 3, 0, 3, 3, 3, 3, 3, 3, 1, 3, 3, 3, 2, 3, 3, 0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 3, 3, 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 3, 3, 3, 3, 3, 3, 1, 0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 1, 3, 0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 3, 3, 3, 3, 0, 3, 3, 2, 3, 3, 3, 3, 3, 3, 0, 3, 3, 0, 3, 3, 0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 3, 0, 3, 3, 3, 3, 0, 3, 0, 1, 3, 3, 3, 3, 3, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 1, 3, 3, 1, 3, 3, 3, 3, 0, 2, 3, 0, 3, 3, 3, 3, 3, 1, 3, 3, 3, 1, 3, 1, 3, 3, 3, 0, 0, 3, 3, 3, 3, 3, 3, 2, 3, 3, 3, 3, 3, 1, 3, 3, 3, 3, 3, 3, 1, 3, 3, 3, 0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 3, 3, 3, 2, 3, 3, 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, 0, 3, 3, 3, 0, 3, 3, 3, 3, 0, 3, 3, 1, 3, 3, 3, 3, 3, 0, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 3, 3, 3, 3, 3, 3, 3, 1, 1, 3, 3, 3, 2, 3, 3, 3, 3, 3, 0, 3, 3, 3, 3, 0, 3, 3, 0, 3, 3, 3, 3, 3, 3]\n",
      "Epoch 1/1\n",
      "1026/1026 [==============================] - 52s 50ms/step - loss: 3.1812 - acc: 0.5419\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/Tim/anaconda/lib/python3.6/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.00      0.00      0.00        32\n",
      "          1       0.00      0.00      0.00        18\n",
      "          2       0.00      0.00      0.00        10\n",
      "          3       0.81      1.00      0.90       261\n",
      "\n",
      "avg / total       0.66      0.81      0.73       321\n",
      "\n",
      "0.81308411215\n",
      "Accuracy: 81.31%\n"
     ]
    }
   ],
   "source": [
    "# Create the embedding layer from the embedding matrix\n",
    "\n",
    "def create_cnn(embedding_weights=embedding_weights,embedding_dim=300,max_sequence_len=800,filters=300,k=6,dropout=0.9):    \n",
    "    embedding_layer = Embedding(vocab_size,\n",
    "                                EMBEDDING_DIM,\n",
    "                                weights=[embedding_weights],\n",
    "                                input_length=MAX_SEQUENCE_LENGTH,\n",
    "                                trainable=False)\n",
    "\n",
    "    # Build the layers of the model\n",
    "    sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32') \n",
    "    embedded_sequences = embedding_layer(sequence_input) \n",
    "\n",
    "    x = Conv1D(FILTERS, k, activation='relu')(embedded_sequences) \n",
    "    x = MaxPooling1D(k)(x)\n",
    "    x = Dropout(DROPOUT)(x)\n",
    "    x = Conv1D(FILTERS, k, activation='relu')(x)\n",
    "    x = MaxPooling1D(k)(x)\n",
    "    x = Dropout(DROPOUT)(x)\n",
    "    x = Conv1D(FILTERS, k, activation='relu')(x)\n",
    "    x = MaxPooling1D(int(x.shape[1]))(x)  # This layer pools the entire previous layer\n",
    "    x = Flatten()(x)\n",
    "    x = Dense(FILTERS, activation='relu')(x)\n",
    "    preds = Dense(len(possibilities), activation='softmax')(x)\n",
    "    \n",
    "    model = Model(sequence_input, preds)\n",
    "    model.compile(loss='categorical_crossentropy',\n",
    "                  optimizer='rmsprop',\n",
    "                  metrics=['acc'])\n",
    "    return model\n",
    "\n",
    "# Train the model\n",
    "# model = create_cnn(embedding_weights,EMBEDDING_DIM,MAX_SEQUENCE_LENGTH,FILTERS,k,DROPOUT)\n",
    "# model.fit(x_train, y_train,epochs=1, batch_size=FILTERS)\n",
    "# model.evaluate(x_final_test,y_final_test,verbose=0)\n",
    "\n",
    "def make_classifications_list(binary_targets):\n",
    "    \"\"\"This is  for turning the output of Keras models into a list of class integers\"\"\"\n",
    "    y_true = list(binary_targets)\n",
    "    for i in range(len(y_true)):\n",
    "        classification = None\n",
    "        for j in range(y_true[i].shape[0]):\n",
    "            if y_true[i][j] == 1.:\n",
    "                classification = j\n",
    "        y_true[i] = classification\n",
    "    return y_true\n",
    "\n",
    "model = create_cnn(embedding_weights,EMBEDDING_DIM,MAX_SEQUENCE_LENGTH,FILTERS,k,DROPOUT)\n",
    "model.fit(x_train, y_train,epochs=1, batch_size=FILTERS)\n",
    "y_prob = model.predict(x_final_test,batch_size=FILTERS)\n",
    "y_pred = y_prob.argmax(axis=-1) # Get the predicted class (not probabilites of each)\n",
    "ypred = list(y_pred) # Turn array into list\n",
    "y_true = make_classifications_list(y_final_test) # Turn Matrix of targets into list\n",
    "print(classification_report(y_true, y_pred))\n",
    "print(accuracy_score(y_true,y_pred))\n",
    "scores = model.evaluate(x_final_test,y_final_test,verbose=0)\n",
    "print(\"Accuracy: %.2f%%\" % (scores[1]*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_90 (Embedding)     (None, 700, 300)          11065200  \n",
      "_________________________________________________________________\n",
      "dropout_141 (Dropout)        (None, 700, 300)          0         \n",
      "_________________________________________________________________\n",
      "lstm_9 (LSTM)                (None, 128)               219648    \n",
      "_________________________________________________________________\n",
      "dropout_142 (Dropout)        (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_147 (Dense)            (None, 4)                 516       \n",
      "=================================================================\n",
      "Total params: 11,285,364\n",
      "Trainable params: 220,164\n",
      "Non-trainable params: 11,065,200\n",
      "_________________________________________________________________\n",
      "None\n",
      "Epoch 1/1\n",
      "1026/1026 [==============================] - 45s 43ms/step - loss: 1.1590 - acc: 0.7300\n",
      "Accuracy: 81.31%\n"
     ]
    }
   ],
   "source": [
    "# RNN Implementation Test Case\n",
    "# Best results: 78.12% accuracy, epochs = 1, LSTM = 128, Dropout = 0.5, adam optimized\n",
    "\n",
    "# Initialize embedding layer\n",
    "embedding_layer = Embedding(vocab_size,\n",
    "                            EMBEDDING_DIM,\n",
    "                            weights=[embedding_weights],\n",
    "                            input_length=MAX_SEQUENCE_LENGTH,\n",
    "                            trainable=False)\n",
    "sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32') \n",
    "embedded_sequences = embedding_layer(sequence_input) \n",
    "\n",
    "rnn = Sequential()\n",
    "rnn.add(embedding_layer)\n",
    "rnn.add(Dropout(0.5))\n",
    "rnn.add(LSTM(128))\n",
    "rnn.add(Dropout(0.5))\n",
    "rnn.add(Dense(len(possibilities), activation='sigmoid'))\n",
    "rnn.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "print(rnn.summary())\n",
    "rnn.fit(x_train, y_train, epochs=1, batch_size=64)\n",
    "\n",
    "scores = rnn.evaluate(x_dev, y_dev, verbose=0)\n",
    "print(\"Accuracy: %.2f%%\" % (scores[1]*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}