sigmoid_and_relu_FF.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===========================================================\n",
      "The output is the matrix of shape:  (22, 2)\n",
      "===========================================================\n",
      "[[0.         0.19091548]\n",
      " [1.29076764 0.24878724]\n",
      " [0.         0.19091548]\n",
      " [0.         0.19091548]\n",
      " [0.         0.19091548]\n",
      " [1.61543014 0.25853366]\n",
      " [0.         0.20271051]\n",
      " [0.51106626 0.22538048]\n",
      " [2.14032275 0.27429102]\n",
      " [0.         0.19091548]\n",
      " [0.         0.19091548]\n",
      " [0.         0.19091548]\n",
      " [0.         0.19222035]\n",
      " [0.         0.19091548]\n",
      " [0.         0.20550199]\n",
      " [0.36416935 0.22097062]\n",
      " [0.         0.19091548]\n",
      " [2.10731974 0.27330027]\n",
      " [0.         0.19171842]\n",
      " [0.         0.19261759]\n",
      " [0.         0.19091548]\n",
      " [0.         0.19091548]]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "# You should write your target function in this space\n",
    "#====================================================\n",
    "#sigmoid function produces output values ranging from 0 to 1(probability)\n",
    "def sigmoid(x):\n",
    "     return 1/(1+np.exp(-x))\n",
    "\n",
    "# ReLU stands for rectified linear unit.   \n",
    "# his function takes a single number as an input, returning 0 if the \n",
    "# input is negative, and the input if the input is positive.\n",
    "def relu(x):\n",
    "    return np.maximum(0, x)\n",
    "#=====================================================\n",
    "\n",
    "\n",
    "# The structure of our NN is 5x4x3x2 neurons. \n",
    "# i.e 5 is input neuron-5x4 weights-4 hidden neurons-\n",
    "# 4x3 weights-3 hidden neurons-3x2 weights-2 output\n",
    "# neurons.\n",
    "\n",
    "# weight and bias initialization\n",
    "\n",
    "np.random.seed(1)  \n",
    "# random.seed gives same random values whenever the script is executed\n",
    "# weights of hidden layer 1 is the matrix of:\n",
    "# No. of input neurons X No.of hidden1 neurons\n",
    "wh1 = np.random.randn(5,4) \n",
    "# bias of hidden layer 1 is the matrix of:\n",
    "# 1 X hidden layer1 neuron\n",
    "bh1 = np.random.randn(1,4) \n",
    "# weights of hidden layer 2 is the matrix of:\n",
    "# No. of hidden1 neurons X No.of hidden2 neurons\n",
    "wh2 = np.random.randn(4,3)\n",
    "# bias of hidden layer 2 is the matrix of:\n",
    "# 1 X hidden layer2 neuron\n",
    "bh2 = np.random.randn(1,3) \n",
    "# weights of output layer is the matrix of:\n",
    "# No. of hidden2 neurons X No.of output neurons\n",
    "wout = np.random.randn(3,2)\n",
    "# bias of output layer is the matrix of:\n",
    "# 1 X output layer neuron\n",
    "bout = np.random.randn(1,2) \n",
    "\n",
    "\n",
    "features_and_targets = np.array(   [ [0, 0, 0, 0, 0, 0, 1],\n",
    "                                     [0, 0, 0, 0, 1, 0, 1],\n",
    "                                     [0, 0, 0, 1, 1, 0, 1],\n",
    "                                     [0, 0, 1, 1, 1, 0, 1],\n",
    "                                     [0, 1, 1, 1, 1, 0, 1],\n",
    "                                     [1, 1, 1, 1, 0, 0, 1],\n",
    "                                     [1, 1, 1, 0, 0, 0, 1],\n",
    "                                     [1, 1, 0, 0, 0, 0, 1],\n",
    "                                     [1, 0, 0, 0, 0, 0, 1],\n",
    "                                     [1, 0, 0, 1, 0, 0, 1],\n",
    "                                     [1, 0, 1, 1, 0, 0, 1],\n",
    "                                     [1, 1, 0, 1, 0, 0, 1],\n",
    "                                     [0, 1, 0, 1, 1, 0, 1],\n",
    "                                     [0, 0, 1, 0, 1, 0, 1],\n",
    "                                     [1, 0, 1, 1, 1, 1, 0],\n",
    "                                     [1, 1, 0, 1, 1, 1, 0],\n",
    "                                     [1, 0, 1, 0, 1, 1, 0],\n",
    "                                     [1, 0, 0, 0, 1, 1, 0],\n",
    "                                     [1, 1, 0, 0, 1, 1, 0],\n",
    "                                     [1, 1, 1, 0, 1, 1, 0],\n",
    "                                     [1, 1, 1, 1, 1, 1, 0],\n",
    "                                     [1, 0, 0, 1, 1, 1, 0]  ]\n",
    "                           , dtype=float)\n",
    "\n",
    "# shuffle our cases\n",
    "np.random.shuffle(features_and_targets)\n",
    "\n",
    "features = features_and_targets[:,0:5]\n",
    "targets = features_and_targets[:,5:7]\n",
    "\n",
    "# the feed-forward phase below is where inputs from the previous \n",
    "# layer are multiplied with the corresponding weights & biases \n",
    "# and are passed through the activation function to get the final\n",
    "# value for the corresponding node in the next layer. This \n",
    "# process is repeated for all the hidden layers until the output \n",
    "# is calculated.\n",
    "\n",
    "\n",
    "while True:\n",
    "    select = input(\"Please select sigmoid activation function by typing 's' or select relu activation function by tying 'r': \")\n",
    "    if select in [\"s\",\"r\"]:\n",
    "        break\n",
    "#sigmoid\n",
    "if select == \"s\":\n",
    "    hidden_layer1_input = (np.matmul(features,wh1)) + bh1\n",
    "    activation1 = sigmoid(hidden_layer1_input)\n",
    "    hidden_layer2_input = (np.matmul(activation1,wh2)) + bh2\n",
    "    activation2 = sigmoid(hidden_layer2_input)\n",
    "    output_layer = (np.matmul(activation2,wout)) + bout\n",
    "    activation_output = sigmoid(output_layer)\n",
    "    print(\"===========================================================\")\n",
    "    print(\"The output is the matrix of shape: \",activation_output.shape)\n",
    "    print(\"===========================================================\")\n",
    "    print(activation_output)\n",
    "    \n",
    "    \n",
    "#relu\n",
    "elif select == \"r\":\n",
    "    hidden_layer1_input = (np.matmul(features,wh1)) + bh1\n",
    "    activation1 = relu(hidden_layer1_input)\n",
    "    hidden_layer2_input = (np.matmul(activation1,wh2)) + bh2\n",
    "    activation2 = relu(hidden_layer2_input)\n",
    "    output_layer = (np.matmul(activation2,wout)) + bout\n",
    "    activation_output = relu(output_layer)\n",
    "    print(\"===========================================================\")\n",
    "    print(\"The output is the matrix of shape: \",activation_output.shape)\n",
    "    print(\"===========================================================\")\n",
    "    print(activation_output)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://github.com/Kulbear/deep-learning-nano-foundation/wiki/Introduction-to-Neural-Network:-Feedforward github sigmoid\n",
    "\n",
    "https://towardsdatascience.com/activation-functions-neural-networks-1cbd9f8d91d6 medium all activation functions\n",
    "\n",
    "https://stackoverflow.com/questions/28403782/what-is-the-difference-between-back-propagation-and-feed-forward-neural-network difference between feed forward and back propogation.\n",
    "\n",
    "https://www.analyticsvidhya.com/blog/2017/05/neural-network-from-scratch-in-python-and-r/ activation function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import autograd.numpy as np\n",
    "from autograd import grad\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "sigmoid function produces output values ranging from 0 to 1(probability)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(features,wh1,bh1,wh2,bh2,wout,bout):\n",
    "    hidden_layer1_input = np.matmul(features, wh1) + bh1\n",
    "    activation1 = 1/(1+np.exp(-hidden_layer1_input))\n",
    "    hidden_layer2_input = np.matmul(activation1,wh2) + bh2\n",
    "    activation2 = 1/(1+np.exp(-hidden_layer2_input))\n",
    "    output_layer = np.matmul(activation2,wout) + bout\n",
    "    activation_output = 1/(1+np.exp(-output_layer))\n",
    "    return activation_output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "ReLU stands for rectified linear unit.   \n",
    "This function takes a single number as an input, \n",
    "returning 0 if the input is negative, and the input \n",
    "if the input is positive."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def relu(features,wh1,bh1,wh2,bh2,wout,bout):\n",
    "    hidden_layer1_input = (np.matmul(features,wh1)) + bh1\n",
    "    activation1 = np.maximum(0, hidden_layer1_input)\n",
    "    hidden_layer2_input = np.matmul(activation1,wh2) + bh2\n",
    "    activation2 = np.maximum(0, hidden_layer2_input)\n",
    "    output_layer = np.matmul(activation2,wout) + bout\n",
    "    activation_output = np.maximum(0, output_layer)\n",
    "    return activation_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The structure of our NN is 5x4x3x2 neurons. \n",
    "# i.e 5 is input neuron-5x4 weights-4 hidden neurons-\n",
    "# 4x3 weights-3 hidden neurons-3x2 weights-2 output\n",
    "# neurons.\n",
    "\n",
    "# weight and bias initialization\n",
    "\n",
    "np.random.seed(1)  \n",
    "# random.seed gives same random values whenever the script is executed\n",
    "# weights of hidden layer 1 is the matrix of:\n",
    "# No. of input neurons X No.of hidden1 neurons\n",
    "wh1 = np.random.randn(5,4) \n",
    "# bias of hidden layer 1 is the matrix of:\n",
    "# 1 X hidden layer1 neuron\n",
    "bh1 = np.random.randn(1,4) \n",
    "# weights of hidden layer 2 is the matrix of:\n",
    "# No. of hidden1 neurons X No.of hidden2 neurons\n",
    "wh2 = np.random.randn(4,3)\n",
    "# bias of hidden layer 2 is the matrix of:\n",
    "# 1 X hidden layer2 neuron\n",
    "bh2 = np.random.randn(1,3) \n",
    "# weights of output layer is the matrix of:\n",
    "# No. of hidden2 neurons X No.of output neurons\n",
    "wout = np.random.randn(3,2)\n",
    "# bias of output layer is the matrix of:\n",
    "# 1 X output layer neuron\n",
    "bout = np.random.randn(1,2) \n",
    "\n",
    "\n",
    "features_and_targets = np.array(   [ [0, 0, 0, 0, 0, 0, 1],\n",
    "                                     [0, 0, 0, 0, 1, 0, 1],\n",
    "                                     [0, 0, 0, 1, 1, 0, 1],\n",
    "                                     [0, 0, 1, 1, 1, 0, 1],\n",
    "                                     [0, 1, 1, 1, 1, 0, 1],\n",
    "                                     [1, 1, 1, 1, 0, 0, 1],\n",
    "                                     [1, 1, 1, 0, 0, 0, 1],\n",
    "                                     [1, 1, 0, 0, 0, 0, 1],\n",
    "                                     [1, 0, 0, 0, 0, 0, 1],\n",
    "                                     [1, 0, 0, 1, 0, 0, 1],\n",
    "                                     [1, 0, 1, 1, 0, 0, 1],\n",
    "                                     [1, 1, 0, 1, 0, 0, 1],\n",
    "                                     [0, 1, 0, 1, 1, 0, 1],\n",
    "                                     [0, 0, 1, 0, 1, 0, 1],\n",
    "                                     [1, 0, 1, 1, 1, 1, 0],\n",
    "                                     [1, 1, 0, 1, 1, 1, 0],\n",
    "                                     [1, 0, 1, 0, 1, 1, 0],\n",
    "                                     [1, 0, 0, 0, 1, 1, 0],\n",
    "                                     [1, 1, 0, 0, 1, 1, 0],\n",
    "                                     [1, 1, 1, 0, 1, 1, 0],\n",
    "                                     [1, 1, 1, 1, 1, 1, 0],\n",
    "                                     [1, 0, 0, 1, 1, 1, 0]  ]\n",
    "                           , dtype=float)\n",
    "\n",
    "# shuffle our cases\n",
    "np.random.shuffle(features_and_targets)\n",
    "\n",
    "features = features_and_targets[:,0:5]\n",
    "targets = features_and_targets[:,5:7]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The feed-forward phase below is where inputs from the previous \n",
    "layer are multiplied with the corresponding weights & biases \n",
    "and are passed through the activation function to get the final\n",
    "value for the corresponding node in the next layer. This \n",
    "process is repeated for all the hidden layers until the output\n",
    "is calculated."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Target :  [0. 1.] Prediction :  [0.61853847 0.48850933]\n",
      "Target :  [0. 1.] Prediction :  [0.6261396  0.49883212]\n",
      "Target :  [0. 1.] Prediction :  [0.61370827 0.48493996]\n",
      "Target :  [0. 1.] Prediction :  [0.59935076 0.48089846]\n",
      "Target :  [0. 1.] Prediction :  [0.61805167 0.48692254]\n",
      "Target :  [1. 0.] Prediction :  [0.63550488 0.49544837]\n",
      "Target :  [0. 1.] Prediction :  [0.61026258 0.49152615]\n",
      "Target :  [0. 1.] Prediction :  [0.62442688 0.48978966]\n",
      "Target :  [0. 1.] Prediction :  [0.63135543 0.49859851]\n",
      "Target :  [1. 0.] Prediction :  [0.61036106 0.48614679]\n",
      "Target :  [1. 0.] Prediction :  [0.61093357 0.48447115]\n",
      "Target :  [1. 0.] Prediction :  [0.61522166 0.48599915]\n",
      "Target :  [1. 0.] Prediction :  [0.61854362 0.49260978]\n",
      "Target :  [0. 1.] Prediction :  [0.59360945 0.47749073]\n",
      "Target :  [1. 0.] Prediction :  [0.6216304  0.48845318]\n",
      "Target :  [1. 0.] Prediction :  [0.61737258 0.4882396 ]\n",
      "Target :  [1. 0.] Prediction :  [0.61103172 0.48370993]\n",
      "Target :  [0. 1.] Prediction :  [0.64127458 0.49536522]\n",
      "Target :  [0. 1.] Prediction :  [0.61022392 0.48831483]\n",
      "Target :  [0. 1.] Prediction :  [0.62174039 0.49093717]\n",
      "Target :  [0. 1.] Prediction :  [0.61343117 0.48607453]\n",
      "Target :  [0. 1.] Prediction :  [0.59499034 0.48466235]\n"
     ]
    }
   ],
   "source": [
    "while True:\n",
    "    select = input(\"Please select sigmoid activation function by typing 's' or select relu activation function by tying 'r': \")\n",
    "    if select in [\"s\",\"r\"]:\n",
    "        break\n",
    "#sigmoid\n",
    "if select == \"s\":\n",
    "    for i in range(22):\n",
    "        print('Target : ', targets[i,:], 'Prediction : ', sigmoid(features,wh1,bh1,wh2,bh2,wout,bout)[i,:])\n",
    "        \n",
    "    \n",
    "elif select == \"r\":\n",
    "    for i in range(22):\n",
    "        print('Target : ', targets[i,:], 'Prediction : ', relu(features,wh1,bh1,wh2,bh2,wout,bout)[i,:]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}