MLWARE_TECHNEX22.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "MLWARE_TECHNEX22.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "Team members - Ayush Agarwal ( me) and Anshuman Asuliya \n",
        "\n",
        "This project of ours is a submission for the event \"MLWare\" , a machine learning event held under \"Technex \" , the tech fest of IIT BHU .\n",
        "\n",
        "The problem statment can be seen from : https://www.kaggle.com/c/ml-ware22/overview\n",
        "\n",
        "The data can be referred from : https://www.kaggle.com/c/ml-ware22/data\n",
        "\n"
      ],
      "metadata": {
        "id": "Pt9IHS6PQDcp"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PY9QtzGdP_4H"
      },
      "outputs": [],
      "source": [
        "pip install kaggle "
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os \n",
        "os.environ['KAGGLE_CONFIG_DIR']='/content'\n",
        "!kaggle competitions download -c ml-ware22\n",
        "!unzip test.h5.zip\n",
        "!unzip train.h5.zip"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hN13w8ilr1ET",
        "outputId": "e83af422-20c2-433e-f34f-08d1c0620cb4"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Warning: Your Kaggle API key is readable by other users on this system! To fix this, you can run 'chmod 600 /content/kaggle.json'\n",
            "Warning: Looks like you're using an outdated API Version, please consider updating (server 1.5.12 / client 1.5.4)\n",
            "Downloading sample_submission.csv to /content\n",
            "  0% 0.00/44.0k [00:00<?, ?B/s]\n",
            "100% 44.0k/44.0k [00:00<00:00, 38.8MB/s]\n",
            "Downloading test.h5.zip to /content\n",
            " 99% 1.21G/1.23G [00:09<00:00, 84.3MB/s]\n",
            "100% 1.23G/1.23G [00:09<00:00, 136MB/s] \n",
            "Downloading train.h5.zip to /content\n",
            "100% 2.49G/2.49G [00:14<00:00, 177MB/s]\n",
            "100% 2.49G/2.49G [00:14<00:00, 190MB/s]\n",
            "Archive:  test.h5.zip\n",
            "  inflating: test.h5                 \n",
            "Archive:  train.h5.zip\n",
            "  inflating: train.h5                \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import h5py\n",
        "ds = h5py.File('train.h5', 'r')\n",
        "x1 = ds['x']\n",
        "x = x1\n",
        "y = ds['y']\n",
        "ds2 = h5py.File('test.h5' , 'r')\n",
        "x_test = ds2['x']\n",
        "y_pred = []"
      ],
      "metadata": {
        "id": "QwBlu3uzr64h"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "##########################################\n",
        "# BLOB DETECTOR CODE - DOES NOT WORK \n",
        "#########################################\n",
        "# import cv2\n",
        "# import numpy as np;\n",
        "# import random \n",
        "# from google.colab.patches import cv2_imshow\n",
        "# detector = cv2.SimpleBlobDetector()\n",
        "# # keypoints = detector.detect(im)\n",
        "# for j in [0,1,2,3,4]:\n",
        "#    i=random.randrange(0,1000)\n",
        "#    cv2_imshow(cv2.resize(x[i],(0,0),fx=0.5,fy=0.5))\n",
        "#    print(y[i])\n",
        "#    keypoints = detector.detect(x[i,:,:,:])\n",
        "#    print(len(keypoints))\n",
        "####################################"
      ],
      "metadata": {
        "id": "uZm_nLAosgt2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "###############################################\n",
        "# IMAGE PROCESSING AND BLOB DETECTION METHOD FOR THE PS \n",
        "###############################################\n",
        "from skimage.measure import label, regionprops, regionprops_table\n",
        "import cv2\n",
        "from google.colab.patches import cv2_imshow\n",
        "import random\n",
        "import numpy as np \n",
        "from skimage.io import imread, imshow\n",
        "from skimage.color import rgb2gray\n",
        "from skimage.feature import blob_dog, blob_log, blob_doh\n",
        "from math import sqrt\n",
        "import matplotlib.pyplot as plt\n",
        "# import numpy as np\n",
        "for j in [0,1,2,3,4]:\n",
        "  i=random.randrange(0,1000)\n",
        "\n",
        "  #first the normal image showing \n",
        "  print(\"normal image\")\n",
        "  cv2_imshow(cv2.resize(x[i],(0,0),fx=0.5,fy=0.5))\n",
        "\n",
        "  #showing the number of blue dots \n",
        "  print(\"number of blue dots as \")\n",
        "  print(\"per the data given to us \")\n",
        "  print(y[i])\n",
        "\n",
        "  lower_purple = np.array([110,150,150])\n",
        "  upper_purple = np.array([170,255,255])\n",
        "  purple_mask = cv2.inRange(x[i],lower_purple,upper_purple)\n",
        "  print(\"Masking output \")\n",
        "  cv2_imshow(cv2.resize(purple_mask,(0,0),fx=0.5,fy=0.5))\n",
        "\n",
        "  #getting the blue channel out of the pic and printing it \n",
        "  #cv2_imshow(cv2.resize(x[i,:,:,0],(0,0),fx=0.5,fy=0.5))\n",
        "  print(\"blue channel of the image \")\n",
        "  cv2_imshow(cv2.resize(x[i,:,:,2],(0,0),fx=0.5,fy=0.5))\n",
        "\n",
        "  # #taking the not of the image \n",
        "  img1 = cv2.bitwise_not(x[i,:,:,2])\n",
        "  print(\"After the first bitwise not \")\n",
        "  cv2_imshow(cv2.resize(img1,(0,0),fx=0.5,fy=0.5))\n",
        "\n",
        "  # #applying the required threshold\n",
        "  print(\"After the first thresholding \")\n",
        "  ret, thresh1 = cv2.threshold(img1, 140, 255, cv2.THRESH_TOZERO)\n",
        "  cv2_imshow(cv2.resize(thresh1,(0,0),fx=0.5,fy=0.5))\n",
        "\n",
        "  # blackening the borders \n",
        "  shapea = thresh1.shape\n",
        "  for a in range(0, shapea[0]):\n",
        "    for b in range(0, 15):\n",
        "            thresh1[a,b] = 0\n",
        "  for a in range(0, shapea[0]):\n",
        "    for b in range(shapea[1]-15,shapea[1]):\n",
        "            thresh1[a,b] = 0\n",
        "  for a in range(0, 15):\n",
        "    for b in range(0, shapea[1]):\n",
        "            thresh1[a,b] = 0\n",
        "  for a in range(shapea[0]-15, shapea[0]):\n",
        "    for b in range(0,shapea[1]):\n",
        "            thresh1[a,b] = 0\n",
        "  \n",
        "  print(\"After removal of the borders\")\n",
        "  cv2_imshow(cv2.resize(thresh1,(0,0),fx=0.5,fy=0.5))\n",
        "\n",
        "  ret, thresh2 = cv2.threshold(thresh1, 110, 255, cv2.THRESH_BINARY)\n",
        "  print(\"After binary thresholding \")\n",
        "  cv2_imshow(cv2.resize(thresh2,(0,0),fx=0.5,fy=0.5))\n",
        "\n",
        "  img2 = label(thresh2)\n",
        "  #print(len(img2))\n",
        "  img2_rp = regionprops(img2)\n",
        "  list1 = []\n",
        "  for u in img2_rp:\n",
        "    if(u.area > 140):\n",
        "      list1.append(u.area)\n",
        "  print(\"value predicted by model is :\")\n",
        "  print(len(list1))\n",
        "\n",
        "###########################################\n",
        "  #OPENCV CONTOUR CODE , DOES NOT WORK \n",
        "  # img2 = multi_ero(multi_dil(thresh1,5),5)\n",
        "  #cv2_imshow(cv2.resize(img2,(0,0),fx=0.5,fy=0.5))\n",
        "  # thresh1 = cv2.bitwise_not(thresh1)\n",
        "  # contours1,_=cv2.findContours(thresh1,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)\n",
        "  # cv2.drawContours(thresh1,contours1,-1,(0,255,0),3)\n",
        "  # cv2_imshow(cv2.resize(thresh1,(0,0),fx=0.5,fy=0.5))\n",
        "###############################################"
      ],
      "metadata": {
        "id": "PbHBwCY1tDUt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# ##########################################\n",
        "# # CODE FOR GENERATING PREDICTIONS FROM THE BLOB AND IMAGE PROCESSING ALGORITHM \n",
        "# DONT RUN UNLESS OUTPUT FILE NEEDED \n",
        "# import h5py\n",
        "# ds = h5py.File('test.h5' , 'r')\n",
        "# x = ds['x']\n",
        "# y_pred = []\n",
        "\n",
        "# import cv2\n",
        "# from google.colab.patches import cv2_imshow\n",
        "# import random\n",
        "# import numpy as np \n",
        "# from skimage.io import imread, imshow\n",
        "# from skimage.color import rgb2gray\n",
        "# from skimage.feature import blob_dog, blob_log, blob_doh\n",
        "# from math import sqrt\n",
        "# import matplotlib.pyplot as plt\n",
        "# from skimage.measure import label, regionprops, regionprops_table\n",
        "\n",
        "# from sklearn.metrics import mean_squared_error\n",
        "\n",
        "# def cell(area):\n",
        "#   y_pred = []\n",
        "#   for i in range(0,x.shape[0]):\n",
        "#     # #taking the not of the image \n",
        "#     img1 = cv2.bitwise_not(x[i,:,:,2])\n",
        "#     img1 = cv2.GaussianBlur(img1 , (7,7) , cv2.BORDER_DEFAULT)\n",
        "\n",
        "#     # #applying the required threshold\n",
        "#     ret, thresh1 = cv2.threshold(img1, 140, 255, cv2.THRESH_TOZERO)\n",
        "\n",
        "#     # blackening the borders \n",
        "#     shapea = thresh1.shape\n",
        "#     for a in range(0, shapea[0]):\n",
        "#       for b in range(0, 15):\n",
        "#               thresh1[a,b] = 0\n",
        "#     for a in range(0, shapea[0]):\n",
        "#       for b in range(shapea[1]-15,shapea[1]):\n",
        "#               thresh1[a,b] = 0\n",
        "#     for a in range(0, 15):\n",
        "#       for b in range(0, shapea[1]):\n",
        "#               thresh1[a,b] = 0\n",
        "#     for a in range(shapea[0]-15, shapea[0]):\n",
        "#       for b in range(0,shapea[1]):\n",
        "#               thresh1[a,b] = 0\n",
        "\n",
        "#     ret, thresh2 = cv2.threshold(thresh1, 110, 255, cv2.THRESH_BINARY)\n",
        "#     img2 = label(thresh2)\n",
        "#     #print(len(img2))\n",
        "#     img2_rp = regionprops(img2)\n",
        "#     list1 = []\n",
        "#     for u in img2_rp:\n",
        "#       if(u.area > area):\n",
        "#         list1.append(u.area)\n",
        "\n",
        "#     y_pred.append(len(list1))\n",
        "#   # print(mean_squared_error(y,y_pred))\n",
        "#   return y_pred\n",
        "\n",
        "# y_pred = cell(200)\n",
        "# id = list(range(0,x.shape[0]))\n",
        "\n",
        "# import pandas as pd\n",
        "\n",
        "# dict = {'id': id, 'label': y_pred}  \n",
        "       \n",
        "# result = pd.DataFrame(dict) \n",
        "    \n",
        "# # saving the dataframe \n",
        "# result.to_csv('file.csv', index=False)\n",
        "\n",
        "# from google.colab import files\n",
        "# files.download(\"file.csv\")\n",
        "####################################################"
      ],
      "metadata": {
        "id": "A2CzeMyf3iTI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "####################################################\n",
        "# THE PURE ML WAY , USING CNNs \n",
        "\n",
        "# converting to numpy\n",
        "X_train = np.array(X_train)\n",
        "# reshaping test data\n",
        "print(X_train.shape)\n",
        "Y_train = np.array(Y_train)\n",
        "print(Y_train.shape)\n",
        "\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras import datasets, layers, models\n",
        "\n",
        "# dividing test data in batches\n",
        "train_ds = tf.data.Dataset.from_tensor_slices((X_train, Y_train)).shuffle(2000).batch(1000)\n",
        "\n",
        "# The CNN model which was used for the final predictions \n",
        "# model\n",
        "cnn = models.Sequential([\n",
        "    layers.Conv2D(filters=32, kernel_size=(3, 3), activation='relu', input_shape=(299, 299, 1)),\n",
        "    layers.MaxPooling2D((2, 2)),\n",
        "    \n",
        "    layers.Conv2D(filters=64, kernel_size=(3, 3), activation='relu'),\n",
        "    layers.MaxPooling2D((2, 2)),\n",
        "    \n",
        "    layers.Flatten(),\n",
        "    layers.Dense(64, activation='relu'),\n",
        "    layers.Dense(1, activation='relu')\n",
        "])\n",
        "print(cnn.summary())\n",
        "\n",
        "# compiling model\n",
        "cnn.compile(optimizer='adam',loss='mean_squared_error',metrics=['accuracy'])\n",
        "\n",
        "# training model of all batched \n",
        "for images,label in train_ds:\n",
        "  cnn.fit(images , label , epochs=10)\n",
        "\n",
        "fname = \"weights-Test-CNN.h5\"\n",
        "cnn.save_weights(fname,overwrite=True)\n",
        "files.download(\"weights-Test-CNN.h5\")\n",
        "\n",
        "\n",
        "# test data\n",
        "test = h5py.File('test.h5' , 'r')\n",
        "x_test = test['x']\n",
        "\n",
        "#test data after threshold\n",
        "X_test = []\n",
        "\n",
        "# convert ti numpy array\n",
        "X_test = np.array(X_test)\n",
        "# reshaping test data\n",
        "X_test = np.reshape(X_test , (X_test.shape[0] , 299,299,1))\n",
        "print(X_test.shape)\n",
        "\n",
        "# prediction\n",
        "y_pred = cnn.predict(X_test)\n",
        "\n",
        "# converting to numpy\n",
        "y_pred = np.array(y_pred)\n",
        "print(y_pred.shape)\n",
        "\n",
        "# round off predected values\n",
        "y_pred = np.round(y_pred)\n",
        "# reshaping predicted data\n",
        "y_pred = y_pred.flatten()\n",
        "print(y_pred)\n",
        "\n",
        "# id column\n",
        "id = list(range(0,x_test.shape[0]))\n",
        "\n",
        "dict = {'id': id, 'label': y_pred}  \n",
        "       \n",
        "result = pd.DataFrame(dict) \n",
        "    \n",
        "# saving the dataframe \n",
        "result.to_csv('file.csv', index=False)\n",
        "\n",
        "from google.colab import files\n",
        "files.download(\"file.csv\")\n",
        "\n",
        "##################################################"
      ],
      "metadata": {
        "id": "G8xkt6L7KORC"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}