hw5_mlp_backprop.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8f7bb785",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[2.3943e-05, 3.7064e-05, 4.2687e-06, 6.3700e-06],\n",
      "        [3.4104e-05, 5.2794e-05, 6.0804e-06, 9.0735e-06],\n",
      "        [2.4438e-05, 3.7831e-05, 4.3571e-06, 6.5019e-06],\n",
      "        [2.0187e-05, 3.1250e-05, 3.5991e-06, 5.3707e-06]])\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch import nn, diag, inverse, transpose\n",
    "\n",
    "\n",
    "def sigma(x):\n",
    "    return torch.sigmoid(x)\n",
    "def sigma_prime(x):\n",
    "    return sigma(x)*(1-sigma(x))\n",
    "\n",
    "\n",
    "torch.manual_seed(0)\n",
    "L = 6\n",
    "X_data = torch.rand(4, 1)\n",
    "Y_data = torch.rand(1, 1)\n",
    "\n",
    "A_list,b_list = [],[]\n",
    "for _ in range(L-1):\n",
    "    A_list.append(torch.rand(4, 4))\n",
    "    b_list.append(torch.rand(4, 1))\n",
    "A_list.append(torch.rand(1, 4))\n",
    "b_list.append(torch.rand(1, 1))\n",
    "\n",
    "\n",
    "\n",
    "'''\n",
    "# Option 1: directly use PyTorch's autograd feature\n",
    "for A in A_list:\n",
    "    A.requires_grad = True\n",
    "for b in b_list:\n",
    "    b.requires_grad = True\n",
    "    \n",
    "y = X_data\n",
    "for ell in range(L):\n",
    "    S = sigma if ell<L-1 else lambda x: x\n",
    "    y = S(A_list[ell]@y+b_list[ell])\n",
    "    \n",
    "# backward pass in pytorch\n",
    "loss=torch.square(y-Y_data)/2\n",
    "loss.backward()\n",
    "\n",
    "print(A_list[0].grad)\n",
    "'''\n",
    "\n",
    "'''\n",
    "# Option 2: construct a NN model and use backprop\n",
    "class MLP(nn.Module) :\n",
    "    def __init__(self) :\n",
    "        super().__init__()\n",
    "        self.linear = nn.ModuleList([nn.Linear(4,4) for _ in range(L-1)])\n",
    "        self.linear.append(nn.Linear(4,1))\n",
    "        for ell in range(L):\n",
    "            self.linear[ell].weight.data = A_list[ell]\n",
    "            self.linear[ell].bias.data = b_list[ell].squeeze()\n",
    "        \n",
    "    def forward(self, x) :\n",
    "        x = x.squeeze()\n",
    "        for ell in range(L-1):\n",
    "            x = sigma(self.linear[ell](x))\n",
    "        x = self.linear[-1](x)\n",
    "        return x\n",
    "\n",
    "model = MLP()\n",
    "            \n",
    "loss = torch.square(model(X_data)-Y_data)/2\n",
    "loss.backward()\n",
    "\n",
    "print(model.linear[0].weight.grad)\n",
    "'''\n",
    "\n",
    "\n",
    "# Option 3: implement backprop yourself\n",
    "y_list = [X_data]\n",
    "y = X_data\n",
    "for ell in range(L):\n",
    "    S = sigma if ell<L-1 else lambda x: x\n",
    "    y = S(A_list[ell]@y+b_list[ell])\n",
    "    y_list.append(y)\n",
    "\n",
    "dA_list = []\n",
    "db_list = []\n",
    "\n",
    "dy = y-Y_data  # dloss/dy_L\n",
    "\n",
    "for ell in reversed(range(L)):\n",
    "    S = sigma_prime if ell<L-1 else lambda x: torch.ones(x.shape)\n",
    "    A, b, y = A_list[ell], b_list[ell], y_list[ell]\n",
    "    #print(\"---------------\")\n",
    "    #print(ell)\n",
    "    #print(f'A shape : {A.shape}')\n",
    "    #print(f'b shape : {b.shape}')\n",
    "    #print(f'y shape : {y.shape}')\n",
    "    #print(f'dy shape : {dy.shape}')\n",
    "    #print(\"--------calculate--------\")\n",
    "    db = transpose(diag(S(A@y+b).squeeze(-1)), 0, 1) @ dy   # dloss/db_ell\n",
    "    #print(f'db shape : {db.shape}')\n",
    "\n",
    "    dA = diag(S(A@y+b).squeeze(-1)) @ dy @ transpose(y, 0, 1)   # dloss/dA_ell\n",
    "    #print(f'dA shape : {dA.shape}')\n",
    "\n",
    "    dy = transpose(diag(S(A@y+b).squeeze(-1)) @ A, 0, 1) @ dy  # dloss/dy_{ell-1}\n",
    "    #print(f'dy shape : {dy.shape}')\n",
    "\n",
    "    dA_list.insert(0, dA)\n",
    "    db_list.insert(0, db)\n",
    "\n",
    "print(dA_list[0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "731e99c3",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}