bi-lstm-with-crf-for-ner.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport keras\nprint(keras.__version__)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\nfrom math import nan\nfrom keras.callbacks import ModelCheckpoint\n\n!pip install git+https://www.github.com/keras-team/keras-contrib.git\nfrom keras_contrib.layers import CRF\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n\n# Any results you write to the current directory are saved as output.","metadata":{"_cell_guid":"4d5be342-abe9-4c95-98fd-c904762ecca9","_uuid":"47570c242f4774c5d670093b822cf6a3f3a91ba9","execution":{"iopub.status.busy":"2021-05-21T12:15:05.694314Z","iopub.execute_input":"2021-05-21T12:15:05.694892Z","iopub.status.idle":"2021-05-21T12:15:18.631776Z","shell.execute_reply.started":"2021-05-21T12:15:05.694844Z","shell.execute_reply":"2021-05-21T12:15:18.62773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## Importing the dataset for named entity recognition model","metadata":{"_cell_guid":"46eeeccc-37ad-4d58-b0c5-71dcbb4b935b","_uuid":"64d208ee42a7c5919fc1f7f0937b7b6fcb9c3ea7"}},{"cell_type":"code","source":"dframe = pd.read_csv(\"../input/entity-annotated-corpus/ner.csv\", encoding = \"ISO-8859-1\", error_bad_lines=False)","metadata":{"_cell_guid":"fab9a876-dd53-4636-a48e-3a87a5ba9370","_uuid":"39f851aae0e531c23978926c6ad9bc3d2c739705","execution":{"iopub.status.busy":"2021-05-21T12:15:18.633236Z","iopub.status.idle":"2021-05-21T12:15:18.633761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dframe","metadata":{"_cell_guid":"0049ec5e-3b99-4059-a09c-7395f1fa3ab7","_uuid":"ce5d3d143151edbe4885ea10ca1994fb2687038a","execution":{"iopub.status.busy":"2021-05-21T12:15:18.634951Z","iopub.status.idle":"2021-05-21T12:15:18.635615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data preprocessing","metadata":{"_cell_guid":"06f00bc1-4964-46e9-a1ff-aa04d2605cef","_uuid":"d64e89af977c30f36cb50f590645ead8576d17ab"}},{"cell_type":"code","source":"dframe.columns","metadata":{"_cell_guid":"85bfa964-b5c4-44d9-9cdc-9eb15259d7c9","_uuid":"028f9857d08bd173ac2a4b587ebd6975aa3c3e80","execution":{"iopub.status.busy":"2021-05-21T12:15:18.636908Z","iopub.status.idle":"2021-05-21T12:15:18.637382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## We want word, pos, sentence_idx and tag as an input ","metadata":{"_cell_guid":"e3448ee3-bed0-4029-88a1-3a79abfd23d5","_uuid":"2d4d560f8101eb4da1a90c05cfa30b32b134be10"}},{"cell_type":"code","source":"dataset=dframe.drop(['Unnamed: 0', 'lemma', 'next-lemma', 'next-next-lemma', 'next-next-pos',\n       'next-next-shape', 'next-next-word', 'next-pos', 'next-shape',\n       'next-word', 'prev-iob', 'prev-lemma', 'prev-pos',\n       'prev-prev-iob', 'prev-prev-lemma', 'prev-prev-pos', 'prev-prev-shape',\n       'prev-prev-word', 'prev-shape', 'prev-word',\"pos\"],axis=1)","metadata":{"_cell_guid":"9f70ec09-0452-495d-aa80-a6358c2d30af","_uuid":"45e7b565a3eacf62a3104969026eed96c7440b34","execution":{"iopub.status.busy":"2021-05-21T12:15:18.638625Z","iopub.status.idle":"2021-05-21T12:15:18.639256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset.info()","metadata":{"_cell_guid":"fbf18f0c-635d-495a-81a2-46cffd0a1141","_uuid":"fc7378e8db3c58bb9f953abf4991b0166a582373","execution":{"iopub.status.busy":"2021-05-21T12:15:18.640318Z","iopub.status.idle":"2021-05-21T12:15:18.640935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset.head()","metadata":{"_cell_guid":"8d992370-b7b0-46d1-939c-74c567c23162","_uuid":"ab6642a1268a00928aa8651037ed84db5414b5f0","execution":{"iopub.status.busy":"2021-05-21T12:15:18.64189Z","iopub.status.idle":"2021-05-21T12:15:18.64238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset=dataset.drop(['shape'],axis=1)","metadata":{"_cell_guid":"4d20e995-322b-4928-85e1-38385b8e8753","_uuid":"8d47f338cf116e591449949652de2f09a1d402c3","execution":{"iopub.status.busy":"2021-05-21T12:15:18.643698Z","iopub.status.idle":"2021-05-21T12:15:18.644221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset.head()","metadata":{"_cell_guid":"768177ab-ed36-46d2-ba06-49b708dbe185","_uuid":"744cc63a695f486351f214f272307656c77db95d","execution":{"iopub.status.busy":"2021-05-21T12:15:18.645273Z","iopub.status.idle":"2021-05-21T12:15:18.645734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create list of list of tuples to differentiate each sentence from each other","metadata":{"_cell_guid":"ba6f0e4d-4b1e-40fa-a72a-df2502841ae4","_uuid":"e5c4b55af4bce4308671e8a611c65bf64d0f0678"}},{"cell_type":"code","source":"class SentenceGetter(object):\n    \n    def __init__(self, dataset):\n        self.n_sent = 1\n        self.dataset = dataset\n        self.empty = False\n        agg_func = lambda s: [(w, t) for w,t in zip(s[\"word\"].values.tolist(),\n                                                        s[\"tag\"].values.tolist())]\n        self.grouped = self.dataset.groupby(\"sentence_idx\").apply(agg_func)\n        self.sentences = [s for s in self.grouped]\n    \n    def get_next(self):\n        try:\n            s = self.grouped[\"Sentence: {}\".format(self.n_sent)]\n            self.n_sent += 1\n            return s\n        except:\n            return None","metadata":{"_cell_guid":"6cab4e05-47c2-4c5b-bcbb-f899f0639691","_uuid":"198ccf9cc26eca32fd8e17b7f3e41bf0af3612b1","execution":{"iopub.status.busy":"2021-05-21T12:15:18.646949Z","iopub.status.idle":"2021-05-21T12:15:18.647398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"getter = SentenceGetter(dataset)","metadata":{"_cell_guid":"16006628-505a-4d37-a5f6-db6a428517b4","_uuid":"fb62acd5794185e2d7714e76712af2ed2ccc6c1d","execution":{"iopub.status.busy":"2021-05-21T12:15:18.648392Z","iopub.status.idle":"2021-05-21T12:15:18.648907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sentences = getter.sentences","metadata":{"_cell_guid":"5d0f958e-ac9c-41cb-a4bc-0256525d4f21","_uuid":"88c8e7bc1b13eac9cffc3a8c4e17b27f116e5776","execution":{"iopub.status.busy":"2021-05-21T12:15:18.650014Z","iopub.status.idle":"2021-05-21T12:15:18.650646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(sentences[5])","metadata":{"_cell_guid":"e33a8086-5906-45b4-8091-2ff97a48d180","_uuid":"d8ef748ceed29d08cab896e10e173ffa750d402c","execution":{"iopub.status.busy":"2021-05-21T12:15:18.651608Z","iopub.status.idle":"2021-05-21T12:15:18.652037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"maxlen = max([len(s) for s in sentences])\nprint ('Maximum sequence length:', maxlen)","metadata":{"_cell_guid":"88cb5e40-fe8a-47a9-9be2-a23340c28a5c","_uuid":"2f4479487f075f8c196a7af9ffaf5dd943a1326d","execution":{"iopub.status.busy":"2021-05-21T12:15:18.653007Z","iopub.status.idle":"2021-05-21T12:15:18.65341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check how long sentences are so that we can pad them\nimport matplotlib.pyplot as plt\n%matplotlib inline\nplt.style.use(\"ggplot\")","metadata":{"_cell_guid":"263e86f4-2605-4b1f-b3df-e3c42f1c9ec4","_uuid":"ef272f6ecc6b8637ebe644b1982f17c41f190039","execution":{"iopub.status.busy":"2021-05-21T12:15:18.654399Z","iopub.status.idle":"2021-05-21T12:15:18.654865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist([len(s) for s in sentences], bins=50)\nplt.show()","metadata":{"_cell_guid":"61e67c66-cd9f-481f-a1ea-a78815dfed47","_uuid":"f25daa9b0767a04690f14ba99ed5730c71cb2100","execution":{"iopub.status.busy":"2021-05-21T12:15:18.655984Z","iopub.status.idle":"2021-05-21T12:15:18.656408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"words = list(set(dataset[\"word\"].values))\nwords.append(\"ENDPAD\")","metadata":{"_cell_guid":"4efea2e0-276c-44bb-b0ce-c94ac99cbf91","_uuid":"db68187a5ea707569031c83db34ca4284025bb8b","execution":{"iopub.status.busy":"2021-05-21T12:15:18.657348Z","iopub.status.idle":"2021-05-21T12:15:18.657739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_words = len(words); n_words","metadata":{"_cell_guid":"5b3697b0-0e7d-4265-9fd1-8001792e6fa5","_uuid":"139a721c0b0036e9cc97892ab8d3d2c3ea0ebafa","execution":{"iopub.status.busy":"2021-05-21T12:15:18.658718Z","iopub.status.idle":"2021-05-21T12:15:18.659132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fix the tags","metadata":{}},{"cell_type":"code","source":"tags = []\nfor tag in set(dataset[\"tag\"].values):\n    if tag is nan or isinstance(tag, float):\n        tags.append('unk')\n    else:\n        tags.append(tag)\nprint(tags)","metadata":{"_cell_guid":"2dc974c3-e5df-45dd-9aff-f8ec43e52399","_uuid":"b51ed46f6c6956fc92a71033c82d804455a01e3d","execution":{"iopub.status.busy":"2021-05-21T12:15:18.660172Z","iopub.status.idle":"2021-05-21T12:15:18.660555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_tags = len(tags); n_tags","metadata":{"_cell_guid":"6cd182c7-7f68-420b-b10f-121da695184e","_uuid":"a426a4acd9481ef8618830c9265f4887d63ec9b5","execution":{"iopub.status.busy":"2021-05-21T12:15:18.661795Z","iopub.status.idle":"2021-05-21T12:15:18.662178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Converting words to numbers and numbers to words**","metadata":{"_cell_guid":"15e92312-4539-490c-a34a-8d8c91e97e0e","_uuid":"8d7134b27205443861225d5820d2ac9e0049e30e"}},{"cell_type":"code","source":"from future.utils import iteritems\nword2idx = {w: i for i, w in enumerate(words)}\ntag2idx = {t: i for i, t in enumerate(tags)}\nidx2tag = {v: k for k, v in iteritems(tag2idx)}","metadata":{"_cell_guid":"805f61f0-ec6b-4afc-a1f6-60a5fe015023","_uuid":"b8109f08b55281540b01c07d95666b2f413aa0a8","execution":{"iopub.status.busy":"2021-05-21T12:15:18.663251Z","iopub.status.idle":"2021-05-21T12:15:18.663735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word2idx['Obama']","metadata":{"_cell_guid":"36b9e479-e006-4d4d-9d0b-fbaf6c89db77","_uuid":"28bf99df1f14a53181bb3eade1ebf68f1d2f8cf8","execution":{"iopub.status.busy":"2021-05-21T12:15:18.664804Z","iopub.status.idle":"2021-05-21T12:15:18.66518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tag2idx[\"O\"]","metadata":{"_cell_guid":"7e614013-5dd7-4309-b481-7a12b881d8f6","_uuid":"8bca49f998176bfa05a21ebd22822066063184b1","execution":{"iopub.status.busy":"2021-05-21T12:15:18.666186Z","iopub.status.idle":"2021-05-21T12:15:18.666694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tag2idx","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:15:18.667776Z","iopub.status.idle":"2021-05-21T12:15:18.668227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx2tag[5]","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:15:18.669304Z","iopub.status.idle":"2021-05-21T12:15:18.669814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx2tag","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:15:18.670769Z","iopub.status.idle":"2021-05-21T12:15:18.671215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.preprocessing.sequence import pad_sequences\nX = [[word2idx[w[0]] for w in s] for s in sentences]","metadata":{"_cell_guid":"5cd7d9ad-77ae-49ba-b99e-6429a949dede","_uuid":"b25bb0979a6bbe8c6eb11b05c23109592d318a5b","execution":{"iopub.status.busy":"2021-05-21T12:15:18.67218Z","iopub.status.idle":"2021-05-21T12:15:18.672772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.array(X).shape","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:15:18.673951Z","iopub.status.idle":"2021-05-21T12:15:18.674344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = pad_sequences(maxlen=140, sequences=X, padding=\"post\",value=n_words - 1)","metadata":{"_cell_guid":"c3ff7a53-4e1b-4b82-bf2e-d581dbdc4315","_uuid":"75ab049dc0c8a6408913da28666c21ac6f9385d3","execution":{"iopub.status.busy":"2021-05-21T12:15:18.675291Z","iopub.status.idle":"2021-05-21T12:15:18.675684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_idx = [[tag2idx[w[1]] for w in s] for s in sentences]\nprint(sentences[100])\nprint(y_idx[100])","metadata":{"_cell_guid":"5d331c3d-a729-454d-ba92-2f3540e3ecac","_uuid":"50607669ece7f12df2be2836193540a0375d848d","execution":{"iopub.status.busy":"2021-05-21T12:15:18.676776Z","iopub.status.idle":"2021-05-21T12:15:18.677355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = pad_sequences(maxlen=140, sequences=y_idx, padding=\"post\", value=tag2idx[\"O\"])\nprint(y_idx[100])","metadata":{"_cell_guid":"8ee02d57-7dc2-4516-9f3b-42507d1d7369","_uuid":"61183c55adee810a0b161cbdfa2a6024862251a4","execution":{"iopub.status.busy":"2021-05-21T12:15:18.678318Z","iopub.status.idle":"2021-05-21T12:15:18.67871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.utils import to_categorical\ny = [to_categorical(i, num_classes=n_tags) for i in y]","metadata":{"_cell_guid":"75b64d49-10a1-411c-85bc-5861a257670e","_uuid":"bd9a39278a2010d51ff4f4a9ccbf0df1d6ee5612","execution":{"iopub.status.busy":"2021-05-21T12:15:18.679703Z","iopub.status.idle":"2021-05-21T12:15:18.680125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)","metadata":{"_cell_guid":"b33bf24e-22cb-4cf4-b504-e84f0ccc84e5","_uuid":"75c39949b2cce357b15f5c42f73421a4b0116de5","execution":{"iopub.status.busy":"2021-05-21T12:15:18.681086Z","iopub.status.idle":"2021-05-21T12:15:18.681713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import Keras","metadata":{}},{"cell_type":"code","source":"from keras.models import Model, Input\nfrom keras.layers import LSTM, Embedding, Dense, TimeDistributed, Dropout, Bidirectional\nimport keras as k","metadata":{"_cell_guid":"00a15ead-883e-4348-96df-e86dcb65a945","_uuid":"89375c4a136104777c96a13b02d2810d78bcc1d9","execution":{"iopub.status.busy":"2021-05-21T12:15:18.682674Z","iopub.status.idle":"2021-05-21T12:15:18.683259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Keras version","metadata":{}},{"cell_type":"code","source":"print(k.__version__)","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:15:18.684347Z","iopub.status.idle":"2021-05-21T12:15:18.684785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model\n **Pay attention to the word embedding size","metadata":{}},{"cell_type":"code","source":"input = Input(shape=(140,))\nword_embedding_size = 300\nmodel = Embedding(input_dim=n_words, output_dim=word_embedding_size, input_length=140)(input)\nmodel = Bidirectional(LSTM(units=word_embedding_size, \n                           return_sequences=True, \n                           dropout=0.5, \n                           recurrent_dropout=0.5, \n                           kernel_initializer=k.initializers.he_normal()))(model)\nmodel = LSTM(units=word_embedding_size * 2, \n             return_sequences=True, \n             dropout=0.5, \n             recurrent_dropout=0.5, \n             kernel_initializer=k.initializers.he_normal())(model)\nmodel = TimeDistributed(Dense(n_tags, activation=\"relu\"))(model)  # previously softmax output layer\n\ncrf = CRF(n_tags)  # CRF layer\nout = crf(model)  # output","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:15:18.685886Z","iopub.status.idle":"2021-05-21T12:15:18.686326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Model(input, out)","metadata":{"_cell_guid":"82ffe0e4-30f4-49a7-902a-e1942ca59259","_uuid":"d9ec4a0fd80b49f9bb28a25de022ec848113fa9a","execution":{"iopub.status.busy":"2021-05-21T12:15:18.687265Z","iopub.status.idle":"2021-05-21T12:15:18.687639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"adam = k.optimizers.Adam(lr=0.0005, beta_1=0.9, beta_2=0.999)\n#model.compile(optimizer=adam, loss=\"categorical_crossentropy\", metrics=[\"accuracy\"])\nmodel.compile(optimizer=adam, loss=crf.loss_function, metrics=[crf.accuracy, 'accuracy'])","metadata":{"_cell_guid":"bf8e202b-f374-42c0-9a8f-146d595c8648","_uuid":"93300d6dc8ac67eecd8f99f2ee82d2817daf5f96","execution":{"iopub.status.busy":"2021-05-21T12:15:18.688637Z","iopub.status.idle":"2021-05-21T12:15:18.689042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:15:18.690016Z","iopub.status.idle":"2021-05-21T12:15:18.690504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Save the model after each epoch if validation is better","metadata":{}},{"cell_type":"code","source":"# Saving the best only\nfilepath=\"ner-bi-lstm-td-model-{val_acc:.2f}.hdf5\"\ncheckpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max')\ncallbacks_list = [checkpoint]","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:15:18.69164Z","iopub.status.idle":"2021-05-21T12:15:18.692077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fit","metadata":{}},{"cell_type":"code","source":"history = model.fit(X_train, np.array(y_train), batch_size=256, epochs=20, validation_split=0.2, verbose=1, callbacks=callbacks_list)","metadata":{"_cell_guid":"5df5d6eb-2285-4a57-b5a0-d0bcbf79c483","_uuid":"93e241a9304d00294cb0f675f1e42b5dbb6a1c07","scrolled":true,"execution":{"iopub.status.busy":"2021-05-21T12:15:18.69312Z","iopub.status.idle":"2021-05-21T12:15:18.693512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Accumulate metrics by tag ","metadata":{}},{"cell_type":"code","source":"TP = {}\nTN = {}\nFP = {}\nFN = {}\nfor tag in tag2idx.keys():\n    TP[tag] = 0\n    TN[tag] = 0    \n    FP[tag] = 0    \n    FN[tag] = 0    \n\ndef accumulate_score_by_tag(gt, pred):\n    \"\"\"\n    For each tag keep stats\n    \"\"\"\n    if gt == pred:\n        TP[gt] += 1\n    elif gt != 'O' and pred == 'O':\n        FN[gt] +=1\n    elif gt == 'O' and pred != 'O':\n        FP[gt] += 1\n    else:\n        TN[gt] += 1\n","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:15:18.69457Z","iopub.status.idle":"2021-05-21T12:15:18.695187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Single prediction and verbose results","metadata":{}},{"cell_type":"code","source":"i = 357\np = model.predict(np.array([X_test[i]]))\np = np.argmax(p, axis=-1)\ngt = np.argmax(y_test[i], axis=-1)\nprint(gt)\nprint(\"{:14}: ({:5}): {}\".format(\"Word\", \"True\", \"Pred\"))\nfor idx, (w,pred) in enumerate(zip(X_test[i],p[0])):\n    #\n    print(\"{:14}: ({:5}): {}\".format(words[w],idx2tag[gt[idx]],tags[pred]))","metadata":{"_uuid":"139269e019aaac2da260b0ee5a676a4c618f6673","execution":{"iopub.status.busy":"2021-05-21T12:15:18.696193Z","iopub.status.idle":"2021-05-21T12:15:18.696773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predict everything at once","metadata":{}},{"cell_type":"code","source":"p = model.predict(np.array(X_test))  ","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:15:18.697894Z","iopub.status.idle":"2021-05-21T12:15:18.698347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## The output is 3d: sent x word x tag prob (softmax)","metadata":{}},{"cell_type":"code","source":"p.shape","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:15:18.699495Z","iopub.status.idle":"2021-05-21T12:15:18.699993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Standard Classification Report","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import classification_report","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:15:18.700937Z","iopub.status.idle":"2021-05-21T12:15:18.701322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Grab the 3d dimension and return the index of the highest probability ... the index matches the tag value\nnp.argmax(p, axis=2)","metadata":{}},{"cell_type":"code","source":"np.argmax(p, axis=2)[0]","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:15:18.702351Z","iopub.status.idle":"2021-05-21T12:15:18.702865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(classification_report(np.argmax(y_test, 2).ravel(), np.argmax(p, axis=2).ravel(),labels=list(idx2tag.keys()), target_names=list(idx2tag.values())))","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:15:18.703971Z","iopub.status.idle":"2021-05-21T12:15:18.704371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Accumulate the scores by tag","metadata":{}},{"cell_type":"code","source":"for i, sentence in enumerate(X_test):\n    y_hat = np.argmax(p[i], axis=-1)\n    gt = np.argmax(y_test[i], axis=-1)\n    for idx, (w,pred) in enumerate(zip(sentence,y_hat)):\n        accumulate_score_by_tag(idx2tag[gt[idx]],tags[pred])","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:15:18.705642Z","iopub.status.idle":"2021-05-21T12:15:18.706237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## How did Classification perform for each tag","metadata":{}},{"cell_type":"code","source":"for tag in tag2idx.keys():\n    print(f'tag:{tag}')    \n    print('\\t TN:{:10}\\tFP:{:10}'.format(TN[tag],FP[tag]))\n    print('\\t FN:{:10}\\tTP:{:10}'.format(FN[tag],TP[tag]))    ","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:15:18.707348Z","iopub.status.idle":"2021-05-21T12:15:18.707738Z"},"trusted":true},"execution_count":null,"outputs":[]}]}