Deep Learning Projects/imdb-movie-review-sentiment-analysis-nlp.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# IMDB Movie Review Sentiment Classification\n\n**Dataset Name:** ACLImdb Movie Review Sentiment Analysis Dataset\n\n**Dataset Description:** The ACLImdb dataset is a benchmark dataset commonly used for sentiment analysis tasks, particularly in the domain of natural language processing (NLP). It consists of movie reviews collected from the IMDb website, spanning a wide range of genres, ratings, and sentiments. Each review is labeled with its corresponding sentiment, indicating whether the review expresses a positive or negative sentiment towards the movie. ","metadata":{}},{"cell_type":"code","source":"import os\nimport keras\nimport tensorflow as tf\nimport numpy as np\nfrom keras import layers","metadata":{"execution":{"iopub.status.busy":"2024-05-26T09:45:45.855693Z","iopub.execute_input":"2024-05-26T09:45:45.856102Z","iopub.status.idle":"2024-05-26T09:45:45.862498Z","shell.execute_reply.started":"2024-05-26T09:45:45.856070Z","shell.execute_reply":"2024-05-26T09:45:45.860853Z"},"trusted":true},"execution_count":35,"outputs":[]},{"cell_type":"markdown","source":"### Loading The Raw Dataset","metadata":{}},{"cell_type":"code","source":"# Download and extract the dataset using these commands\n!curl -O https://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz && tar -xf aclImdb_v1.tar.gz","metadata":{"execution":{"iopub.status.busy":"2024-05-26T07:50:04.979650Z","iopub.execute_input":"2024-05-26T07:50:04.980465Z","iopub.status.idle":"2024-05-26T07:50:20.333770Z","shell.execute_reply.started":"2024-05-26T07:50:04.980423Z","shell.execute_reply":"2024-05-26T07:50:20.332412Z"},"trusted":true},"execution_count":3,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.10/pty.py:89: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock.\n  pid, fd = os.forkpty()\n","output_type":"stream"},{"name":"stdout","text":"  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n                                 Dload  Upload   Total   Spent    Left  Speed\n100 80.2M  100 80.2M    0     0  12.3M      0  0:00:06  0:00:06 --:--:-- 13.0M\n","output_type":"stream"}]},{"cell_type":"code","source":"!ls aclImdb/train","metadata":{"execution":{"iopub.status.busy":"2024-05-26T07:50:20.335931Z","iopub.execute_input":"2024-05-26T07:50:20.336371Z","iopub.status.idle":"2024-05-26T07:50:21.461355Z","shell.execute_reply.started":"2024-05-26T07:50:20.336331Z","shell.execute_reply":"2024-05-26T07:50:21.460182Z"},"trusted":true},"execution_count":4,"outputs":[{"name":"stdout","text":"labeledBow.feat  pos\tunsupBow.feat  urls_pos.txt\nneg\t\t unsup\turls_neg.txt   urls_unsup.txt\n","output_type":"stream"}]},{"cell_type":"code","source":"# remove the unnecessay files\n!rm -r aclImdb/train/unsup","metadata":{"execution":{"iopub.status.busy":"2024-05-26T07:50:21.463281Z","iopub.execute_input":"2024-05-26T07:50:21.463743Z","iopub.status.idle":"2024-05-26T07:50:23.775069Z","shell.execute_reply.started":"2024-05-26T07:50:21.463706Z","shell.execute_reply":"2024-05-26T07:50:23.773413Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"code","source":"# create train, and val set using tensorflow\nbatch_size = 32\nraw_train_ds, raw_val_ds = keras.utils.text_dataset_from_directory(\"aclImdb/train\",\n                                                      batch_size = batch_size,\n                                                      validation_split = 0.2,\n                                                      subset = \"both\",\n                                                      seed = 42)  # add seed or don't shuffle","metadata":{"execution":{"iopub.status.busy":"2024-05-26T07:50:23.780559Z","iopub.execute_input":"2024-05-26T07:50:23.781022Z","iopub.status.idle":"2024-05-26T07:50:26.320442Z","shell.execute_reply.started":"2024-05-26T07:50:23.780983Z","shell.execute_reply":"2024-05-26T07:50:26.319005Z"},"trusted":true},"execution_count":6,"outputs":[{"name":"stdout","text":"Found 25000 files belonging to 2 classes.\nUsing 20000 files for training.\nUsing 5000 files for validation.\n","output_type":"stream"}]},{"cell_type":"code","source":"# create test set\nraw_test_ds = keras.utils.text_dataset_from_directory(\"aclImdb/test\", batch_size = batch_size)","metadata":{"execution":{"iopub.status.busy":"2024-05-26T07:50:26.322003Z","iopub.execute_input":"2024-05-26T07:50:26.322423Z","iopub.status.idle":"2024-05-26T07:50:28.822028Z","shell.execute_reply.started":"2024-05-26T07:50:26.322388Z","shell.execute_reply":"2024-05-26T07:50:28.820414Z"},"trusted":true},"execution_count":7,"outputs":[{"name":"stdout","text":"Found 25000 files belonging to 2 classes.\n","output_type":"stream"}]},{"cell_type":"code","source":"print(f\"Number of batches in raw_train_ds: {raw_train_ds.cardinality()}\")\nprint(f\"Number of batches in raw_val_ds: {raw_val_ds.cardinality()}\")\nprint(f\"Number of batches in raw_test_ds: {raw_test_ds.cardinality()}\")","metadata":{"execution":{"iopub.status.busy":"2024-05-26T07:50:28.823769Z","iopub.execute_input":"2024-05-26T07:50:28.824197Z","iopub.status.idle":"2024-05-26T07:50:28.833158Z","shell.execute_reply.started":"2024-05-26T07:50:28.824168Z","shell.execute_reply":"2024-05-26T07:50:28.831272Z"},"trusted":true},"execution_count":8,"outputs":[{"name":"stdout","text":"Number of batches in raw_train_ds: 625\nNumber of batches in raw_val_ds: 157\nNumber of batches in raw_test_ds: 782\n","output_type":"stream"}]},{"cell_type":"markdown","source":"This means in each batch we have 32 elements (reviews and target). We can check them this way.","metadata":{}},{"cell_type":"code","source":"for text_batch, label_batch in raw_train_ds.take(1):\n    for i in range(2):\n        print(\"text:\")\n        print(text_batch.numpy()[i])\n        print(\"/n\")\n        print(\"Label:\")\n        print(label_batch.numpy()[i])","metadata":{"execution":{"iopub.status.busy":"2024-05-26T07:50:28.835417Z","iopub.execute_input":"2024-05-26T07:50:28.835830Z","iopub.status.idle":"2024-05-26T07:50:28.930400Z","shell.execute_reply.started":"2024-05-26T07:50:28.835799Z","shell.execute_reply":"2024-05-26T07:50:28.928897Z"},"trusted":true},"execution_count":9,"outputs":[{"name":"stdout","text":"text:\nb'\"Pandemonium\" is a horror movie spoof that comes off more stupid than funny. Believe me when I tell you, I love comedies. Especially comedy spoofs. \"Airplane\", \"The Naked Gun\" trilogy, \"Blazing Saddles\", \"High Anxiety\", and \"Spaceballs\" are some of my favorite comedies that spoof a particular genre. \"Pandemonium\" is not up there with those films. Most of the scenes in this movie had me sitting there in stunned silence because the movie wasn\\'t all that funny. There are a few laughs in the film, but when you watch a comedy, you expect to laugh a lot more than a few times and that\\'s all this film has going for it. Geez, \"Scream\" had more laughs than this film and that was more of a horror film. How bizarre is that?<br /><br />*1/2 (out of four)'\n/n\nLabel:\n0\ntext:\nb\"David Mamet is a very interesting and a very un-equal director. His first movie 'House of Games' was the one I liked best, and it set a series of films with characters whose perspective of life changes as they get into complicated situations, and so does the perspective of the viewer.<br /><br />So is 'Homicide' which from the title tries to set the mind of the viewer to the usual crime drama. The principal characters are two cops, one Jewish and one Irish who deal with a racially charged area. The murder of an old Jewish shop owner who proves to be an ancient veteran of the Israeli Independence war triggers the Jewish identity in the mind and heart of the Jewish detective.<br /><br />This is were the flaws of the film are the more obvious. The process of awakening is theatrical and hard to believe, the group of Jewish militants is operatic, and the way the detective eventually walks to the final violent confrontation is pathetic. The end of the film itself is Mamet-like smart, but disappoints from a human emotional perspective.<br /><br />Joe Mantegna and William Macy give strong performances, but the flaws of the story are too evident to be easily compensated.\"\n/n\nLabel:\n0\n","output_type":"stream"}]},{"cell_type":"markdown","source":"### Preparing The Data - Normalization\n\nWe are now going to remove some of the tags present in the text and lowercase.","metadata":{}},{"cell_type":"code","source":"import string\nimport re\n\ndef custom_standardization(input_data):\n    lowercase = tf.strings.lower(input_data)\n    stripped_html = tf.strings.regex_replace(lowercase, \"<br />\", \" \")\n    return tf.strings.regex_replace(\n        stripped_html, f\"[{re.escape(string.punctuation)}]\", \"\"\n    )","metadata":{"execution":{"iopub.status.busy":"2024-05-26T07:50:28.931560Z","iopub.execute_input":"2024-05-26T07:50:28.931924Z","iopub.status.idle":"2024-05-26T07:50:28.939410Z","shell.execute_reply.started":"2024-05-26T07:50:28.931896Z","shell.execute_reply":"2024-05-26T07:50:28.937626Z"},"trusted":true},"execution_count":10,"outputs":[]},{"cell_type":"code","source":"max_features = 20000 # most frequent words\nembedding_dim = 128  \nsequence_length = 500 # sequence length truct\n\nvectorizer_layer = layers.TextVectorization(standardize = custom_standardization,\n                                                 max_tokens = max_features,\n                                                 output_mode = \"int\",\n                                                 output_sequence_length = sequence_length,)\ntext_ds = raw_train_ds.map(lambda x, y: x)\nvectorizer_layer.adapt(text_ds)","metadata":{"execution":{"iopub.status.busy":"2024-05-26T07:50:28.940888Z","iopub.execute_input":"2024-05-26T07:50:28.942794Z","iopub.status.idle":"2024-05-26T07:50:34.898722Z","shell.execute_reply.started":"2024-05-26T07:50:28.942742Z","shell.execute_reply":"2024-05-26T07:50:34.896741Z"},"trusted":true},"execution_count":11,"outputs":[]},{"cell_type":"code","source":"# vectorize the data\ndef vectorize_text(text, label):\n    text = tf.expand_dims(text, -1)\n    return vectorizer_layer(text), label\n\ntrain_ds = raw_train_ds.map(vectorize_text)\nval_ds = raw_val_ds.map(vectorize_text)\ntest_ds = raw_test_ds.map(vectorize_text)","metadata":{"execution":{"iopub.status.busy":"2024-05-26T07:50:34.901133Z","iopub.execute_input":"2024-05-26T07:50:34.901729Z","iopub.status.idle":"2024-05-26T07:50:35.262733Z","shell.execute_reply.started":"2024-05-26T07:50:34.901690Z","shell.execute_reply":"2024-05-26T07:50:35.261254Z"},"trusted":true},"execution_count":12,"outputs":[]},{"cell_type":"code","source":"# for GPU performance\ntrain_ds = train_ds.cache().prefetch(buffer_size = 10)\nval_ds = val_ds.cache().prefetch(buffer_size = 10)\ntest_ds = test_ds.cache().prefetch(buffer_size = 10)","metadata":{"execution":{"iopub.status.busy":"2024-05-26T07:50:35.264473Z","iopub.execute_input":"2024-05-26T07:50:35.265423Z","iopub.status.idle":"2024-05-26T07:50:35.285325Z","shell.execute_reply.started":"2024-05-26T07:50:35.265387Z","shell.execute_reply":"2024-05-26T07:50:35.284262Z"},"trusted":true},"execution_count":13,"outputs":[]},{"cell_type":"markdown","source":"### Create First Model: Baseline","metadata":{}},{"cell_type":"code","source":"from keras.callbacks import EarlyStopping, ReduceLROnPlateau\n\nearly_stopping = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)\nreduce_lr_on_plateu = ReduceLROnPlateau(monitor = \"val_loss\", fraction = 0.5, patience = 2, verbose = 1)\n\ninputs = keras.Input(shape = (None,), dtype = \"int64\")\nx = layers.Embedding(max_features, embedding_dim)(inputs)\nx = layers.Dropout(0.5)(x)\nx = layers.Conv1D(128, 7, padding = 'valid', activation = \"relu\", strides = 3)(x)\nx = layers.Conv1D(128, 7, padding = \"valid\", activation = \"relu\", strides = 3)(x)\nx = layers.GlobalMaxPooling1D()(x)\nx = layers.Dense(128, activation = \"relu\")(x)\nx = layers.Dropout(0.5)(x)\n\noutputs = layers.Dense(1, activation = \"sigmoid\", name = \"predictions\")(x)\n\nmodel = keras.Model(inputs, outputs)\nmodel.compile(loss = \"binary_crossentropy\", optimizer = \"adam\", metrics = [\"accuracy\"])\n\nprint(\"Model Summary:\")\nmodel.summary()\n\nprint(\"Training The Model Now:\")\n\nmodel.fit(train_ds, validation_data=val_ds, epochs= 20, callbacks = [early_stopping, reduce_lr_on_plateu])","metadata":{"execution":{"iopub.status.busy":"2024-05-26T07:50:35.286749Z","iopub.execute_input":"2024-05-26T07:50:35.287100Z","iopub.status.idle":"2024-05-26T07:58:34.320040Z","shell.execute_reply.started":"2024-05-26T07:50:35.287059Z","shell.execute_reply":"2024-05-26T07:58:34.318647Z"},"trusted":true},"execution_count":14,"outputs":[{"name":"stdout","text":"Model Summary:\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"\u001b[1mModel: \"functional_1\"\u001b[0m\n","text/html":"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"functional_1\"</span>\n</pre>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n┃\u001b[1m \u001b[0m\u001b[1mLayer (type)                   \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape          \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m      Param #\u001b[0m\u001b[1m \u001b[0m┃\n┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n│ input_layer (\u001b[38;5;33mInputLayer\u001b[0m)        │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m)           │             \u001b[38;5;34m0\u001b[0m │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ embedding (\u001b[38;5;33mEmbedding\u001b[0m)           │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m128\u001b[0m)      │     \u001b[38;5;34m2,560,000\u001b[0m │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ dropout (\u001b[38;5;33mDropout\u001b[0m)               │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m128\u001b[0m)      │             \u001b[38;5;34m0\u001b[0m │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ conv1d (\u001b[38;5;33mConv1D\u001b[0m)                 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m128\u001b[0m)      │       \u001b[38;5;34m114,816\u001b[0m │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ conv1d_1 (\u001b[38;5;33mConv1D\u001b[0m)               │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m128\u001b[0m)      │       \u001b[38;5;34m114,816\u001b[0m │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ global_max_pooling1d            │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m128\u001b[0m)            │             \u001b[38;5;34m0\u001b[0m │\n│ (\u001b[38;5;33mGlobalMaxPooling1D\u001b[0m)            │                        │               │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ dense (\u001b[38;5;33mDense\u001b[0m)                   │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m128\u001b[0m)            │        \u001b[38;5;34m16,512\u001b[0m │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ dropout_1 (\u001b[38;5;33mDropout\u001b[0m)             │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m128\u001b[0m)            │             \u001b[38;5;34m0\u001b[0m │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ predictions (\u001b[38;5;33mDense\u001b[0m)             │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1\u001b[0m)              │           \u001b[38;5;34m129\u001b[0m │\n└─────────────────────────────────┴────────────────────────┴───────────────┘\n","text/html":"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n┃<span style=\"font-weight: bold\"> Layer (type)                    </span>┃<span style=\"font-weight: bold\"> Output Shape           </span>┃<span style=\"font-weight: bold\">       Param # </span>┃\n┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n│ input_layer (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)        │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>)           │             <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ embedding (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Embedding</span>)           │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span>)      │     <span style=\"color: #00af00; text-decoration-color: #00af00\">2,560,000</span> │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ dropout (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dropout</span>)               │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span>)      │             <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ conv1d (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Conv1D</span>)                 │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span>)      │       <span style=\"color: #00af00; text-decoration-color: #00af00\">114,816</span> │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ conv1d_1 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Conv1D</span>)               │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span>)      │       <span style=\"color: #00af00; text-decoration-color: #00af00\">114,816</span> │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ global_max_pooling1d            │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span>)            │             <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">GlobalMaxPooling1D</span>)            │                        │               │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ dense (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)                   │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span>)            │        <span style=\"color: #00af00; text-decoration-color: #00af00\">16,512</span> │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ dropout_1 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dropout</span>)             │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span>)            │             <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ predictions (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)             │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1</span>)              │           <span style=\"color: #00af00; text-decoration-color: #00af00\">129</span> │\n└─────────────────────────────────┴────────────────────────┴───────────────┘\n</pre>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"\u001b[1m Total params: \u001b[0m\u001b[38;5;34m2,806,273\u001b[0m (10.71 MB)\n","text/html":"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">2,806,273</span> (10.71 MB)\n</pre>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m2,806,273\u001b[0m (10.71 MB)\n","text/html":"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">2,806,273</span> (10.71 MB)\n</pre>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n","text/html":"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n</pre>\n"},"metadata":{}},{"name":"stdout","text":"Training The Model Now:\nEpoch 1/20\n\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m62s\u001b[0m 95ms/step - accuracy: 0.5810 - loss: 0.6296 - val_accuracy: 0.8688 - val_loss: 0.3046 - learning_rate: 0.0010\nEpoch 2/20\n\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m57s\u001b[0m 92ms/step - accuracy: 0.8863 - loss: 0.2840 - val_accuracy: 0.8840 - val_loss: 0.2940 - learning_rate: 0.0010\nEpoch 3/20\n\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m82s\u001b[0m 91ms/step - accuracy: 0.9476 - loss: 0.1453 - val_accuracy: 0.8704 - val_loss: 0.4182 - learning_rate: 0.0010\nEpoch 4/20\n\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 88ms/step - accuracy: 0.9758 - loss: 0.0711\nEpoch 4: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m58s\u001b[0m 93ms/step - accuracy: 0.9758 - loss: 0.0711 - val_accuracy: 0.8786 - val_loss: 0.4601 - learning_rate: 0.0010\nEpoch 5/20\n\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m81s\u001b[0m 93ms/step - accuracy: 0.9886 - loss: 0.0298 - val_accuracy: 0.8796 - val_loss: 0.5234 - learning_rate: 1.0000e-04\nEpoch 6/20\n\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 86ms/step - accuracy: 0.9930 - loss: 0.0216\nEpoch 6: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m57s\u001b[0m 91ms/step - accuracy: 0.9930 - loss: 0.0216 - val_accuracy: 0.8808 - val_loss: 0.5527 - learning_rate: 1.0000e-04\nEpoch 7/20\n\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m81s\u001b[0m 90ms/step - accuracy: 0.9956 - loss: 0.0131 - val_accuracy: 0.8804 - val_loss: 0.5616 - learning_rate: 1.0000e-05\n","output_type":"stream"},{"execution_count":14,"output_type":"execute_result","data":{"text/plain":"<keras.src.callbacks.history.History at 0x7cc4672a2a40>"},"metadata":{}}]},{"cell_type":"code","source":"early_stopping.best","metadata":{"execution":{"iopub.status.busy":"2024-05-26T07:58:34.323994Z","iopub.execute_input":"2024-05-26T07:58:34.324385Z","iopub.status.idle":"2024-05-26T07:58:34.333286Z","shell.execute_reply.started":"2024-05-26T07:58:34.324354Z","shell.execute_reply":"2024-05-26T07:58:34.331945Z"},"trusted":true},"execution_count":15,"outputs":[{"execution_count":15,"output_type":"execute_result","data":{"text/plain":"0.2940349280834198"},"metadata":{}}]},{"cell_type":"code","source":"model.evaluate(test_ds)","metadata":{"execution":{"iopub.status.busy":"2024-05-26T07:58:34.334957Z","iopub.execute_input":"2024-05-26T07:58:34.335452Z","iopub.status.idle":"2024-05-26T07:58:51.446513Z","shell.execute_reply.started":"2024-05-26T07:58:34.335409Z","shell.execute_reply":"2024-05-26T07:58:51.445350Z"},"trusted":true},"execution_count":16,"outputs":[{"name":"stdout","text":"\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m17s\u001b[0m 22ms/step - accuracy: 0.8748 - loss: 0.3157\n","output_type":"stream"},{"execution_count":16,"output_type":"execute_result","data":{"text/plain":"[0.31300050020217896, 0.8762800097465515]"},"metadata":{}}]},{"cell_type":"code","source":"model.save(\"model1.h5\")","metadata":{"execution":{"iopub.status.busy":"2024-05-26T08:12:48.600650Z","iopub.execute_input":"2024-05-26T08:12:48.601403Z","iopub.status.idle":"2024-05-26T08:12:48.706406Z","shell.execute_reply.started":"2024-05-26T08:12:48.601364Z","shell.execute_reply":"2024-05-26T08:12:48.705008Z"},"trusted":true},"execution_count":17,"outputs":[]},{"cell_type":"markdown","source":"### Using Bidrectional LSTM","metadata":{}},{"cell_type":"code","source":"inputs = keras.Input(shape = (None,), dtype = \"int64\")\nx = layers.Embedding(max_features, embedding_dim)(inputs)\nx = layers.Bidirectional(layers.LSTM(32, return_sequences = True))(x)\nx = layers.GlobalMaxPool1D()(x)\nx = layers.Dense(20, activation = \"relu\")(x)\nx = layers.Dropout(0.5)(x)\noutputs = layers.Dense(1, activation = \"sigmoid\", name = \"predictions\")(x)\n\nmodel = keras.Model(inputs, outputs)\nmodel.compile(loss = \"binary_crossentropy\", optimizer = \"adam\", metrics = [\"accuracy\"])\n\nprint(\"Model Summary:\")\nmodel.summary()\n\nprint(\"Training The Model Now:\")\n\nmodel.fit(train_ds, validation_data=val_ds, epochs= 20, callbacks = [early_stopping, reduce_lr_on_plateu])","metadata":{"execution":{"iopub.status.busy":"2024-05-26T09:10:31.345126Z","iopub.execute_input":"2024-05-26T09:10:31.345617Z","iopub.status.idle":"2024-05-26T09:30:05.363464Z","shell.execute_reply.started":"2024-05-26T09:10:31.345548Z","shell.execute_reply":"2024-05-26T09:30:05.362389Z"},"trusted":true},"execution_count":19,"outputs":[{"name":"stdout","text":"Model Summary:\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"\u001b[1mModel: \"functional_3\"\u001b[0m\n","text/html":"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"functional_3\"</span>\n</pre>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n┃\u001b[1m \u001b[0m\u001b[1mLayer (type)                   \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape          \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m      Param #\u001b[0m\u001b[1m \u001b[0m┃\n┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n│ input_layer_1 (\u001b[38;5;33mInputLayer\u001b[0m)      │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m)           │             \u001b[38;5;34m0\u001b[0m │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ embedding_1 (\u001b[38;5;33mEmbedding\u001b[0m)         │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m128\u001b[0m)      │     \u001b[38;5;34m2,560,000\u001b[0m │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ bidirectional (\u001b[38;5;33mBidirectional\u001b[0m)   │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m64\u001b[0m)       │        \u001b[38;5;34m41,216\u001b[0m │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ global_max_pooling1d_1          │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m64\u001b[0m)             │             \u001b[38;5;34m0\u001b[0m │\n│ (\u001b[38;5;33mGlobalMaxPooling1D\u001b[0m)            │                        │               │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ dense_1 (\u001b[38;5;33mDense\u001b[0m)                 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m20\u001b[0m)             │         \u001b[38;5;34m1,300\u001b[0m │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ dropout_2 (\u001b[38;5;33mDropout\u001b[0m)             │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m20\u001b[0m)             │             \u001b[38;5;34m0\u001b[0m │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ predictions (\u001b[38;5;33mDense\u001b[0m)             │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1\u001b[0m)              │            \u001b[38;5;34m21\u001b[0m │\n└─────────────────────────────────┴────────────────────────┴───────────────┘\n","text/html":"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n┃<span style=\"font-weight: bold\"> Layer (type)                    </span>┃<span style=\"font-weight: bold\"> Output Shape           </span>┃<span style=\"font-weight: bold\">       Param # </span>┃\n┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n│ input_layer_1 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)      │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>)           │             <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ embedding_1 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Embedding</span>)         │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span>)      │     <span style=\"color: #00af00; text-decoration-color: #00af00\">2,560,000</span> │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ bidirectional (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Bidirectional</span>)   │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)       │        <span style=\"color: #00af00; text-decoration-color: #00af00\">41,216</span> │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ global_max_pooling1d_1          │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)             │             <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">GlobalMaxPooling1D</span>)            │                        │               │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ dense_1 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)                 │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">20</span>)             │         <span style=\"color: #00af00; text-decoration-color: #00af00\">1,300</span> │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ dropout_2 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dropout</span>)             │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">20</span>)             │             <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ predictions (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)             │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1</span>)              │            <span style=\"color: #00af00; text-decoration-color: #00af00\">21</span> │\n└─────────────────────────────────┴────────────────────────┴───────────────┘\n</pre>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"\u001b[1m Total params: \u001b[0m\u001b[38;5;34m2,602,537\u001b[0m (9.93 MB)\n","text/html":"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">2,602,537</span> (9.93 MB)\n</pre>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m2,602,537\u001b[0m (9.93 MB)\n","text/html":"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">2,602,537</span> (9.93 MB)\n</pre>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n","text/html":"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n</pre>\n"},"metadata":{}},{"name":"stdout","text":"Training The Model Now:\nEpoch 1/20\n\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m195s\u001b[0m 305ms/step - accuracy: 0.6830 - loss: 0.5569 - val_accuracy: 0.8870 - val_loss: 0.2803 - learning_rate: 0.0010\nEpoch 2/20\n\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m191s\u001b[0m 306ms/step - accuracy: 0.9085 - loss: 0.2514 - val_accuracy: 0.8886 - val_loss: 0.2968 - learning_rate: 0.0010\nEpoch 3/20\n\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 290ms/step - accuracy: 0.9534 - loss: 0.1501\nEpoch 3: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m191s\u001b[0m 306ms/step - accuracy: 0.9535 - loss: 0.1501 - val_accuracy: 0.8860 - val_loss: 0.3317 - learning_rate: 0.0010\nEpoch 4/20\n\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m190s\u001b[0m 305ms/step - accuracy: 0.9777 - loss: 0.0788 - val_accuracy: 0.8906 - val_loss: 0.3641 - learning_rate: 1.0000e-04\nEpoch 5/20\n\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 290ms/step - accuracy: 0.9850 - loss: 0.0583\nEpoch 5: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m202s\u001b[0m 305ms/step - accuracy: 0.9850 - loss: 0.0583 - val_accuracy: 0.8900 - val_loss: 0.3922 - learning_rate: 1.0000e-04\nEpoch 6/20\n\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m203s\u001b[0m 308ms/step - accuracy: 0.9900 - loss: 0.0440 - val_accuracy: 0.8908 - val_loss: 0.3985 - learning_rate: 1.0000e-05\n","output_type":"stream"},{"execution_count":19,"output_type":"execute_result","data":{"text/plain":"<keras.src.callbacks.history.History at 0x7cc434cd6ad0>"},"metadata":{}}]},{"cell_type":"code","source":"early_stopping.best","metadata":{"execution":{"iopub.status.busy":"2024-05-26T09:31:57.226068Z","iopub.execute_input":"2024-05-26T09:31:57.226534Z","iopub.status.idle":"2024-05-26T09:31:57.234399Z","shell.execute_reply.started":"2024-05-26T09:31:57.226503Z","shell.execute_reply":"2024-05-26T09:31:57.232934Z"},"trusted":true},"execution_count":20,"outputs":[{"execution_count":20,"output_type":"execute_result","data":{"text/plain":"0.2803375720977783"},"metadata":{}}]},{"cell_type":"code","source":"model.evaluate(test_ds)","metadata":{"execution":{"iopub.status.busy":"2024-05-26T09:32:14.220286Z","iopub.execute_input":"2024-05-26T09:32:14.221328Z","iopub.status.idle":"2024-05-26T09:33:36.142721Z","shell.execute_reply.started":"2024-05-26T09:32:14.221290Z","shell.execute_reply":"2024-05-26T09:33:36.141384Z"},"trusted":true},"execution_count":21,"outputs":[{"name":"stdout","text":"\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m53s\u001b[0m 68ms/step - accuracy: 0.8759 - loss: 0.2968\n","output_type":"stream"},{"execution_count":21,"output_type":"execute_result","data":{"text/plain":"[0.290151983499527, 0.8795599937438965]"},"metadata":{}}]}]}