Routing_Path_Visualization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Code from https://github.com/ageron/handson-ml/blob/master/extra_capsnets.ipynb by Aurélien Geron was used as a guide."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt # Plotting library\n",
    "import numpy as np # Algebra library\n",
    "import tensorflow as tf # Tensorflow\n",
    "import os # Folder management library\n",
    "import random # Pseudo random number generator\n",
    "from sklearn.model_selection import train_test_split # To split data into train, test, validation\n",
    "import pickle # Serializing module.\n",
    "from tensorflow.examples.tutorials.mnist import input_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Available CPUs and GPUs on Machine\n",
    "Just shows what devices are available."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.python.client import device_lib\n",
    "print(device_lib.list_local_devices())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reset Graph\n",
    "Reset Tensorflow state."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set Seed for Reproducibility (Optional)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(42)\n",
    "tf.set_random_seed(42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Load Data for Training, Validation, and Testing \n",
    "Run this cell if you will be training the architecture."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist = input_data.read_data_sets(\"../MNIST/\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualize Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlAAAACBCAYAAAAPH4TmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAADthJREFUeJzt3WtsFUUUwPFpwUqLCCIPgyJKDFCjiAUF1CIqKCASbdEIMRAkBaMUET7Iq5hYDaZRqqRiKUSMVBowPHzERwQVkNQgRKNGIKi0DaRQGqlGo9hK/UA4zozdy53bvXtf/9+nMznbeycuW093Zs+mtba2KgAAAIQvPdYTAAAASDQUUAAAAI4ooAAAABxRQAEAADiigAIAAHBEAQUAAOCIAgoAAMARBRQAAICjjgF/H107Yy/Np8/hXMaeX+dSKc5nPODaTB5cm8mlzfPJHSgAAABHFFAAAACOKKAAAAAcUUABAAA4ooACAABwRAEFAADgiAIKAADAEQUUAACAo6AbaQIAELYzZ85IvGDBAiNXVlYmcXV1tZEbNmxYdCeGlMcdKAAAAEcUUAAAAI4ooAAAAByxBwoAEDcaGhqMcVFRkcQVFRWeP3fkyBFjzB6o+FBQUGCMKysrJd6zZ4+Ry8nJCWROfuEOFAAAgCMKKAAAAEcs4SFl1NbWSrxmzRoj9/zzz0uclpZm5FpbWyXOzs42cs8995zEeXl5vswTSDX19fUSl5SUGLlQy3a5ubkSDx8+3P+Jod369etnjP/66y+JDx8+bORYwgMAAEhyFFAAAACOKKAAAAAcpen7OwIQ6JehTWnnPyQscXkuT548aYyXL18u8VtvvSVxY2OjcZx+HYTaA2XnrrzySom/+uorI9ejR49wpx0pv86lUnF0Pv/++2+J77rrLiP3xRdftPkz3bp1M8bffvutxH379vVxdlGV1NemraWlReJ58+ZJ/Oqrr3r+zBNPPGGMV6xYIXFGRoaPs2u3pLw2I7F+/XpjPG3aNInHjx9v5D744INA5hSBNs8nd6AAAAAcUUABAAA4oo2BUmrdunXGWF+mufTSSyU+cOCAcdzIkSMl1h+nRbD0VgJ612KlzHMZ7lJcz549Pb/LXvqrqamReNSoUUbuhx9+CDFrnKMv2Sml1MyZMyX2WrJTSqn7779f4oULFxq5Pn36tHteJ06cMMa9e/du92fiP4sWLZI41LLd7NmzJS4rK4vqnBCsOFt2dcYdKAAAAEcUUAAAAI4ooAAAABzF5R6oDRs2GOOvv/5a4tdff93372tqavLMdez4338ie69Gp06dJM7KyjJygwcPlnjTpk1GLtQeG7h75513JLb3Ntnjc6699lpj/Pnnn0scqv3A7t27jfHtt98u8aFDh847V/zfSy+9ZIz1t7Xb9MfYX3zxRYn1a7E9FixYILG9N3LZsmUS64/dIzzPPPOMMdbPn27OnDnGWG9VgMSzdetWz9yUKVMCnIn/uAMFAADgiAIKAADAUdx0Ip8/f77Er7zyipE7c+ZM9GYUgDvuuMMYV1VVSRyDR6MTvtux3U7i5ptvllhvO6GUuVyqL83ZywL6v7nFixcbOb3FgU1fIrSXC8vLyyWeNWuW52e0Q8J2O/7+++8l1s+fUkr9+eefEnfp0sXI/fLLLxLry+uRsrvHjxs3rs3vUkqp0tJSiaO0hJfw16btyy+/lHjChAlG7tSpUxLrrQpWrVplHJeenpB/5yfstekHfdvNiBEjjNzFF18scV1dnZHLzMyM7sQiRydyAAAAP1BAAQAAOKKAAgAAcBQ3bQzefvttie09T3pLgEjXSG+99VZjrL8GIlLbt2+X+M033zRy+is+PvvsMyOnP7q5ceNGI0eLg/PLzs42xvo+FrsFgVdLgoqKCs+xvV9J3wO1ZcsWIxdqD1ReXl6b3w2lXnjhBYn1PU9KKXXBBRdI/O677xo5P/Y96exH6fV9T/ZrJvz4nZFq9NYP+p4npZS67777JNZfwZSge56g0Vv+2O1/9PMbx3uewsK/VAAAAEcUUAAAAI7iZglvx44dEuuPOCul1NixYyW2H2uOpdzcXImnT59u5O69916JDx48aOT0JT176U/vhIzwDBo0yPln7KW9gQMHSmy3QtAfX9eXnpRSSm8DYi+/huponur279/vmdNbCYwePdrzuH/++Udie5kglJ9++kninTt3eh6Xn59vjK+66qqwvwNnfffdd565goICiS+//PIgpoOAbN68OdZTCAR3oAAAABxRQAEAADiigAIAAHAUN3ugBgwY0GacKPr372+Mi4uLJX7wwQc9f87eU8MeqPbZtWuXMdb3n+l7kuxWCIcOHZJ4+PDhRq6hoUFiu1VBr169JP7www8jmDFsp0+f9szt3btX4qVLl0r8ySef+PLdl112mcT2K31wfu+//74xPn78uMR2W4+JEycGMicEr76+PtZTCAR3oAAAABxRQAEAADiKmyU8wA8bNmwwxnqHcb3lgL0Up+f0JTs7Z7cqKCwslDgnJyeCGaemp59+WuIZM2YYOb3Nx5133mnk9LYD9hsL/KA/Wn/dddf5/vnJzu7Ur5s8ebIxtq9Bv+n/PuhujmjgXxUAAIAjCigAAABHLOH5ZNWqVcZ43759Yf2c/SJVvUPz0KFD2z+xFOe1TBBq+cDOjRo1SuIVK1YYOZbtIlNXV+eZa25ulth+EbduxIgREj/wwANG7tixYxKvXLky7HkNGzYs7GPxf/rLmG12h38/VFdXS1xeXm7kjh49KrH+snqllOrevbvvc0l1+tsAjhw54nlcJG+OiFfcgQIAAHBEAQUAAOCIAgoAAMARe6DU/7umVlZWSlxaWhrRZ4Trjz/+MMb6Y9u//vprRJ+ZyqZOnWqMa2trJW5sbJRY71CulFK///6752c+++yzErPnyR+PPvqoxBkZGWH/3MMPPyxx3759Je7QoYNx3PLly8P6vNtuu80YT5gwIey54KxTp05JvGPHDt8/X/8dae8L1ffa6HtwbPPnzzfGb7zxhj+Tg9DP0549ezyPGzNmTBDTCQR3oAAAABxRQAEAADhKmSW87du3G2O9XcDq1auNXKhHMKNNX9qAO73lQFvjc+wlvCVLlki8bds2I6e/4Nl+YbD+gmKE74orrpB44cKFvn9+586dwzpu7ty5xrhjx5T5leiblpYWiUMthYerqqrKGJeUlEisv/TbBdshoi/cbSzjxo2L8kyCwx0oAAAARxRQAAAAjiigAAAAHCXVgv/hw4eN8WOPPSbxp59+GtFn9uvXT+JLLrnE87ji4mJj3KlTJ4nnzJlj5EKt4/fp08d1iknj5MmTEvfs2TOq32W/TmDz5s0Sjx8/3sh99NFHEustLpRSat68eVGYHdorPd37b0M9d8011wQxnaSWlZUl8cCBA41cqN91v/32m8QbN26UeNasWT7O7qzMzEzfPxMm+/+B50ycONEYJ1MrGO5AAQAAOKKAAgAAcJTwS3h6p/CysjIj9/PPP0t80UUXGbmuXbtK/NRTTxk5fRntlltukVhfznOhf5etS5cuxti+3ZnMdu3aZYz1dgH2Etv69esDmZNSSi1evNgYf/zxxxJH+hg1glVRUeGZu/vuuyW+8cYbg5hOUtNbRtjXrX69FBUVGbmGhgaJa2pqfJ/XkCFDJH755Zd9/3yYvLrQ21tf7LcGJDLuQAEAADiigAIAAHBEAQUAAOAo4fdAVVdXS6zveVJKqUmTJkms769RyvsVH3755ptvJK6trfU87sILLzTG2dnZUZtTPNBbFcyePdvI9e7dW+Ig9zwpZb5J3J5Xa2troHOBO/tVHfoj8jZaT0SPfe289957Eu/du9f370tLS5O4oKDAyOmP1ffq1cv37051J06cMMbNzc0xmknscAcKAADAEQUUAACAo4RfwisvL5d48ODBRm7p0qVBT0f8+OOPEtu3OnVjxowJYjpxY+vWrRLbLQFGjx4d2DwOHDhgjPPz8yW256UvE9iPaSM+2MtD+rJ5RkaGkevevXsgc0pFdhd/fens+PHj7f78KVOmGOOpU6dKnEotYOKB3TG+qampzeP0c5RsuAMFAADgiAIKAADAEQUUAACAo4TfA6XvZ4jlnieb3l7B1q1bN4nnzp0bxHTiRm5ursR2e4CdO3dKXFlZaeT09g5Dhw71/Hy7ZcTu3bsl3rJli8Tbtm0zjtPnou95Usp87P3JJ5/0/G7ETmFhoWfOfo3TTTfdFO3pIAwzZsyQWH/tysyZM43j0tP/+zs/MzMz+hODp6NHj0q8f/9+z+P0vb333HNPVOcUS9yBAgAAcEQBBQAA4Cjhl/DixfXXX2+MDx486Hms/jb4kSNHRm1O8UhfisvLyzNy+rLatGnTjJy+rJaTk+P5+XV1dca4sbFR4lDLdDp7KTjVllkT0enTpz1zN9xwQ4AzgZeVK1ca48cff1ziDh06BD0dRKChoUHiY8eOeR43ffp0iUP9rk103IECAABwRAEFAADgiAIKAADAEXugfFJTU2OMW1paJO7atauR423wZ+mv4VHK3L+0b98+z5+zc/oau90aQc9lZWVJrO/FUkqpRYsWSWzvzUJiY39N7NTX18d6CgiI3qJm0qRJMZxJcLgDBQAA4IgCCgAAwFGaveQRZYF+WbRVVVVJ/Mgjjxi5zp07S7x27Voj99BDD0V3YqH59Uyp7+dSbzlQVFTkedzq1auNcX5+vsQ9evTw/Dm9i/igQYMimWK88fP54IS+Nq+++mpjrC+pZ2RkGLklS5ZIvGzZsqjOy1HcXptwxrWZXNo8n9yBAgAAcEQBBQAA4IgCCgAAwBFtDBw0Nzcb45KSEontfRaTJ0+WOMZ7nhKGvn/ptdde8zwuVA6pqbCw0BgXFxdL3NTUZOTS0/m7EUD78ZsEAADAEQUUAACAI9oYONC7iyulVGlpqcRDhgwxcmPHjg1kThHgUenkwaPSyYVrM3lwbSYX2hgAAAD4gQIKAADAEQUUAACAI/ZApR72WSQP9lkkF67N5MG1mVzYAwUAAOAHCigAAABHQS/hAQAAJDzuQAEAADiigAIAAHBEAQUAAOCIAgoAAMARBRQAAIAjCigAAABHFFAAAACOKKAAAAAcUUABAAA4ooACAABwRAEFAADgiAIKAADAEQUUAACAIwooAAAARxRQAAAAjiigAAAAHFFAAQAAOKKAAgAAcEQBBQAA4IgCCgAAwBEFFAAAgCMKKAAAAEcUUAAAAI7+Bdvhl4PbffAYAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x216 with 5 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Labels: [7 3 4 6 1]\n"
     ]
    }
   ],
   "source": [
    "n_samples = 5\n",
    "\n",
    "plt.figure(figsize=(n_samples * 2, 3))\n",
    "for index in range(n_samples):\n",
    "    plt.subplot(1, n_samples, index + 1)\n",
    "    sample_image = mnist.train.images[index].reshape(28, 28)\n",
    "    plt.imshow(sample_image, cmap=\"binary\")\n",
    "    plt.axis(\"off\")\n",
    "plt.show()\n",
    "\n",
    "# Print labels\n",
    "print('Labels:', mnist.train.labels[0:n_samples])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Define Model Architecture"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predictions(caps_Lminus1, dims_Lminus1, output_Lminus1, \n",
    "                caps_L, dims_L, batch_size, weight_sharing=False, grid_cells_Lminus1=1, name=None):\n",
    "    \n",
    "    with tf.name_scope(name, default_name=\"predictions\"):\n",
    "        if weight_sharing == False:\n",
    "            output_Lminus1 = tf.squeeze(output_Lminus1, axis=[1, 4], name=\"output_Lminus1\")\n",
    "\n",
    "        # Initialize first array to random normal\n",
    "        init_sigma = 0.1\n",
    "        W_init = tf.random_normal(shape=(1, caps_Lminus1, caps_L, dims_L, dims_Lminus1),\n",
    "                                  stddev=init_sigma, dtype=tf.float32, name=\"W_init\")\n",
    "        W = tf.Variable(W_init, name=\"W\")\n",
    "        tf.summary.histogram(\"W\", W)\n",
    "        \n",
    "        # Tile W for the batch\n",
    "        W_tiled = tf.tile(W, [batch_size, grid_cells_Lminus1, 1, 1, 1], name=\"W_tiled\") \n",
    "        # the 1 in the shape argument in tf.tile means keep dimension from W.\n",
    "\n",
    "\n",
    "        # -1 means add another dimension to the end of tensor\n",
    "        output_expanded_Lminus1 = tf.expand_dims(output_Lminus1, -1, name=\"output_expanded_Lminus1\")\n",
    "        # 2 means add another dimension to the 2th position of tensor\n",
    "        output_tile_Lminus1 = tf.expand_dims(output_expanded_Lminus1, 2, name=\"output_tile_Lminus1\")\n",
    "        # 1 in shape argument means keep dimensions from caps1_output_tile\n",
    "        output_tiled_Lminus1 = tf.tile(output_tile_Lminus1, [1, 1, caps_L, 1, 1], name=\"output_tiled_Lminus1\")\n",
    "        # Get prediction tensor (3rd array) by using t.matmul\n",
    "        predicted_L = tf.matmul(W_tiled, output_tiled_Lminus1, name=\"predicted_L\")\n",
    "    \n",
    "        return predicted_L\n",
    "\n",
    "    \n",
    "def routing(caps_Lminus1, caps_L, batch_size, iterations, predicted_L, name=None):\n",
    "    \n",
    "    with tf.name_scope(name, default_name=\"routing\"):\n",
    "        # initialize the raw routing weights to zero\n",
    "        # The two extra 1s is so that raw_weights and weighted_predictions have the same size\n",
    "        raw_weights = tf.zeros([batch_size, caps_Lminus1, caps_L, 1, 1], dtype=np.float32, name=\"raw_weights\")\n",
    "\n",
    "        for r in range(0, iterations):\n",
    "\n",
    "            # c_i = softmax(b_i)\n",
    "            routing_weights = tf.nn.softmax(raw_weights, dim=2, name=\"routing_weights\")\n",
    "            # weighted sum of all the predicted output vectors for each second-layer capsule\n",
    "            weighted_predictions = tf.multiply(routing_weights, predicted_L, name=\"weighted_predictions\")\n",
    "            weighted_sum = tf.reduce_sum(weighted_predictions, axis=(1), keepdims=True, name=\"weighted_sum\")\n",
    "\n",
    "            # Squash weighted_sum (s_j) to get output for 2nd capsules (v_j)\n",
    "            output_L = squash(weighted_sum, axis=-2, name=\"output_L\")\n",
    "\n",
    "            # Make caps2_output same size as predicted_caps\n",
    "            output_tiled_L = tf.tile(output_L, \n",
    "                                         [1, caps_Lminus1, 1, 1, 1], \n",
    "                                         name=\"output_tiled_L\")\n",
    "            # the 1 in the shape argument in tf.tile means keep dimension from caps2_output.\n",
    "\n",
    "            # Dot Product\n",
    "            agreement = tf.matmul(predicted_L, output_tiled_L, transpose_a=True, name=\"agreement\")\n",
    "            # Update raw_weights\n",
    "            raw_weights = tf.add(raw_weights, agreement, name=\"raw_weights\")\n",
    "\n",
    "        return output_L, routing_weights\n",
    "        \n",
    "# safe_norm taken from Aurélien Geron\n",
    "def squash(s, axis=-1, epsilon=1e-7, name=None):\n",
    "    \n",
    "    with tf.name_scope(name, default_name=\"squash\"):\n",
    "        squared_norm = tf.reduce_sum(tf.square(s), axis=axis,\n",
    "                                     keepdims=True)\n",
    "        safe_norm = tf.sqrt(squared_norm + epsilon)\n",
    "        squash_factor = squared_norm / (1. + squared_norm)\n",
    "        unit_vector = s / safe_norm\n",
    "        return squash_factor * unit_vector\n",
    "    \n",
    "    \n",
    "# safe_norm taken from Aurélien Geron\n",
    "def safe_norm(s, axis=-1, epsilon=1e-7, keep_dims=False, name=None):\n",
    "    \n",
    "    with tf.name_scope(name, default_name=\"safe_norm\"):\n",
    "        squared_norm = tf.reduce_sum(tf.square(s), axis=axis,\n",
    "                                     keep_dims=keep_dims)\n",
    "        return tf.sqrt(squared_norm + epsilon)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Input "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Placeholder for the input images (batch size, length, width, greyscale).\n",
    "# Batch size, length, and width are set to None as they can vary. \n",
    "X = tf.placeholder(shape=[None, None, None, 1], dtype=tf.float32, name=\"X\")\n",
    "tf.summary.image('X', X)\n",
    "y = tf.placeholder(shape=[None], dtype=tf.int64, name=\"y\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convolutional Layer\n",
    "Used to get feature map."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyper-parameters\n",
    "conv1_params = {\n",
    "    \"filters\": 256,\n",
    "    \"kernel_size\": 3,\n",
    "    \"strides\": 1,\n",
    "    \"padding\": \"valid\",\n",
    "    \"activation\": tf.nn.relu,\n",
    "}\n",
    "# Convolve image with conv1 filters\n",
    "conv1 = tf.layers.conv2d(X, name=\"conv1\", **conv1_params)\n",
    "tf.summary.histogram(\"conv1\", conv1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ConvCaps Layer\n",
    "Also known as primary capsule layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# No. of dimenions a single capsule contains.\n",
    "convcaps_dims = 8 \n",
    "# No. of capsules per grid cell.\n",
    "convcaps_caps_types = 32 \n",
    "\n",
    "convcaps_params = {\n",
    "    \"filters\": convcaps_caps_types * convcaps_dims,\n",
    "    \"kernel_size\": 3,\n",
    "    \"strides\": 1,\n",
    "    \"padding\": \"valid\",\n",
    "    \"activation\": tf.nn.relu \n",
    "}\n",
    "# Convolve conv1 with filters.\n",
    "convcaps = tf.layers.conv2d(conv1, name=\"convcaps\", **convcaps_params)\n",
    "tf.summary.histogram(\"convcaps\", convcaps)\n",
    "\n",
    "# Length and width of convaps layer.\n",
    "convcaps_grid_length = tf.shape(convcaps)[1]   \n",
    "convcaps_grid_width = tf.shape(convcaps)[2] \n",
    "convcaps_grid_cells = convcaps_grid_length * convcaps_grid_width\n",
    "# Total No. capsules in convcaps layer.\n",
    "convcaps_caps = convcaps_caps_types * convcaps_grid_cells\n",
    "\n",
    "# Reshape convcaps so that it is easier to work with.\n",
    "convcaps_reshape = tf.reshape(convcaps, [-1, convcaps_caps, convcaps_dims], name=\"convcaps_reshape\")\n",
    "# Squash convcaps with squashing function.\n",
    "convcaps_output = squash(convcaps_reshape, name=\"convcaps_output\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### First Capsule Layer\n",
    "The first capsule layer that is generated by the routing algorithm."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The batch size\n",
    "batch_size = tf.shape(X)[0]\n",
    "\n",
    "caps1_caps = 10\n",
    "caps1_dims = 16\n",
    "\n",
    "caps1_predictions = predictions(convcaps_caps_types, convcaps_dims, convcaps_output, \n",
    "                                caps1_caps, caps1_dims, batch_size, True, convcaps_grid_cells, \"caps1_predictions\")\n",
    "\n",
    "caps1_output, routing1 = routing(convcaps_caps, caps1_caps, \n",
    "                                  batch_size, 9, caps1_predictions, \"routing1\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate Accuracy\n",
    "Accuracy given by $$ \\frac{\\text{True Positives} + \\text{True Negatives}}{\\text{Total Dataset Size}}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(name=\"accuracy_cell\"):\n",
    "    \n",
    "    lengths_temp = safe_norm(caps1_output, axis=-2, name=\"lengths_temp\")\n",
    "    lengths = tf.reshape(lengths_temp, shape=(-1, 10),name=\"lengths\")\n",
    "    lengths_argmax = tf.argmax(lengths, axis=1, name=\"lengths_argmax\")\n",
    "    y_predictions = lengths_argmax\n",
    "    correct = tf.equal(y, y_predictions, name=\"correct\")\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name=\"accuracy\")\n",
    "    tf.summary.scalar('accuracy', accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate Loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "m_plus = 0.9\n",
    "m_minus = 0.1\n",
    "lambda_ = 0.5\n",
    "\n",
    "\n",
    "with tf.name_scope(name=\"loss_cell\"):\n",
    "\n",
    "    T = tf.one_hot(y, depth=caps1_caps, name=\"T\")\n",
    "    # Compute norm of each capsule in digitcaps\n",
    "    caps1_output_norm = safe_norm(caps1_output, axis=-2, keep_dims=True, name=\"caps1_output_norm\")\n",
    "    \n",
    "    present_error_raw = tf.square(tf.maximum(0., m_plus - caps1_output_norm), name=\"present_error_raw\")\n",
    "    present_error = tf.reshape(present_error_raw, shape=(-1, caps1_caps), name=\"present_error\")\n",
    "    # -1 tells reshape to calculate the size of this dimension. \n",
    "\n",
    "    absent_error_raw = tf.square(tf.maximum(0., caps1_output_norm - m_minus), name=\"absent_error_raw\")\n",
    "    absent_error = tf.reshape(absent_error_raw, shape=(-1, caps1_caps), name=\"absent_error\")\n",
    "    # -1 tells reshape to calculate the size of this dimension. \n",
    "\n",
    "    # Compute Margin Loss\n",
    "    L = tf.add(T * present_error, lambda_ * (1.0 - T) * absent_error, name=\"L\")\n",
    "    loss = tf.reduce_mean(tf.reduce_sum(L, axis=1), name=\"loss\")\n",
    "    tf.summary.scalar('loss', loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Adam Optimizer\n",
    "Initialize adam optimizer to minimize $\\texttt{loss}$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(name=\"train\"):\n",
    "    learning_rate = tf.placeholder(tf.float32, shape=[])\n",
    "    tf.summary.scalar('learning_rate', learning_rate)\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "    training_op = optimizer.minimize(loss, name=\"training_op\")\n",
    "    \n",
    "merged_summary = tf.summary.merge_all()\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Train the Architecture\n",
    "Run the below cell if you would like to train the architecture."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Learning Rate and Other Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time step:  5700\n",
      "learning rate:  0.001\n"
     ]
    }
   ],
   "source": [
    "# Initial Learning Rate.\n",
    "learning_r = 0.001\n",
    "# Batch size for training.\n",
    "batch_size = 5\n",
    "\n",
    "# Save model and tensorboard data to this folder\n",
    "folder = \"NEW_MNIST\"+str(batch_size)+\"_lr\"+str(learning_r)\n",
    "\n",
    "# Load time step for Tensorboard.\n",
    "if os.path.isfile('./tensorboard/Models/'+folder+'/time_step.pickle'):\n",
    "    time_step_unpickle = open('./tensorboard/Models/'+folder+'/time_step.pickle', 'rb')\n",
    "    time_step = pickle.load(time_step_unpickle)\n",
    "else:\n",
    "    time_step = 0\n",
    "    \n",
    "# Load learning rate    \n",
    "if os.path.isfile('./tensorboard/Models/'+folder+'/learning_r.pickle'):\n",
    "    learning_r_unpickle = open('./tensorboard/Models/'+folder+'/learning_r.pickle', 'rb')\n",
    "    learning_r = pickle.load(learning_r_unpickle)\n",
    "            \n",
    "print('time step: ', time_step)\n",
    "print('learning rate: ', learning_r)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training\n",
    "Training code taken from Aurélien Géron."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 10000\n",
    "restore_checkpoint = True\n",
    "\n",
    "n_iterations_per_epoch = len(mnist.train.images) // batch_size\n",
    "n_iterations_validation = len(mnist.validation.images) // batch_size\n",
    "best_loss_val = np.infty\n",
    "\n",
    "if not os.path.exists(\"./tensorboard/Models/\"+folder):\n",
    "    os.makedirs(\"./tensorboard/Models/\"+folder)\n",
    "checkpoint_path = \"./tensorboard/Models/\"+folder+\"/my_capsule_network\"\n",
    "\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    if restore_checkpoint and tf.train.checkpoint_exists(checkpoint_path):\n",
    "        saver.restore(sess, checkpoint_path)\n",
    "    else:\n",
    "        init.run()\n",
    "    writer = tf.summary.FileWriter(\"./tensorboard/\"+folder)\n",
    "    writer.add_graph(sess.graph, global_step=time_step)\n",
    "\n",
    "    lr_count = 0\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(1, n_iterations_per_epoch + 1)[:300]:\n",
    "            X_batch = mnist.train.images[(batch_size*(iteration-1)):(batch_size*iteration)]\n",
    "            y_batch = mnist.train.labels[(batch_size*(iteration-1)):(batch_size*iteration)]\n",
    "            \n",
    "            # Every 5 iterations display loss\n",
    "            if time_step % 5 == 0:\n",
    "                loss_, merged_summary_, y_predictions_, y_ = sess.run(\n",
    "                    [loss, merged_summary, y_predictions, y],\n",
    "                    feed_dict={X: X_batch.reshape([-1, 28, 28, 1]),\n",
    "                               y: y_batch, \n",
    "                               learning_rate: learning_r})\n",
    "                writer.add_summary(merged_summary_, time_step)\n",
    "                print(\"\\rIteration: {}/{} ({:.1f}%)  Loss: {:.6f}  Prediction: {}  y: {}\".format(\n",
    "                    iteration, n_iterations_per_epoch, iteration * 100 / n_iterations_per_epoch,\n",
    "                    loss_, y_predictions_, y_), end=\"\")\n",
    "            \n",
    "            # Train Model\n",
    "            _ = sess.run([training_op], feed_dict={X: X_batch.reshape([-1, 28, 28, 1]),\n",
    "                                                   y: y_batch, \n",
    "                                                   learning_rate: learning_r})\n",
    "            \n",
    "            time_step = time_step + 1\n",
    "            \n",
    "            \n",
    "            \n",
    "         # At the end of each epoch,\n",
    "        # measure the validation loss and accuracy:\n",
    "        loss_vals = []\n",
    "        acc_vals = []\n",
    "        for iteration in range(1, n_iterations_validation + 1):\n",
    "            X_batch = mnist.validation.images[(batch_size*(iteration-1)):(batch_size*iteration)]\n",
    "            y_batch = mnist.validation.labels[(batch_size*(iteration-1)):(batch_size*iteration)]\n",
    "            loss_val, acc_val = sess.run(\n",
    "                    [loss, accuracy],\n",
    "                    feed_dict={X: X_batch.reshape([-1, 28, 28, 1]),\n",
    "                               y: y_batch})\n",
    "            loss_vals.append(loss_val)\n",
    "            acc_vals.append(acc_val)\n",
    "            print(\"\\rEvaluating the model: {}/{} ({:.1f}%)\".format(\n",
    "                      iteration, n_iterations_validation,\n",
    "                      iteration * 100 / n_iterations_validation),\n",
    "                  end=\" \" * 10)\n",
    "        loss_val = np.mean(loss_vals)\n",
    "        acc_val = np.mean(acc_vals)\n",
    "        print(\"\\rEpoch: {}  Val accuracy: {:.4f}%  Loss: {:.6f}{}\".format(\n",
    "            epoch + 1, acc_val * 100, loss_val,\n",
    "            \" (improved)\" if loss_val < best_loss_val else \"\"))\n",
    "        \n",
    "        # And save the model if it improved:\n",
    "        if loss_val < best_loss_val:\n",
    "            save_path = saver.save(sess, checkpoint_path)\n",
    "            best_loss_val = loss_val\n",
    "        else:\n",
    "            lr_count = lr_count + 1\n",
    "            \n",
    "            if lr_count == 3:\n",
    "                learning_r = learning_r/10\n",
    "                lr_count = 0\n",
    "                print(learning_r)\n",
    "            \n",
    "                with open('./tensorboard/Models/'+folder+'/learning_r.pickle', 'wb') as f:\n",
    "                    pickle.dump(learning_r, f, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "                    \n",
    "                if restore_checkpoint and tf.train.checkpoint_exists(checkpoint_path):\n",
    "                    saver.restore(sess, checkpoint_path)\n",
    "        \n",
    "        with open('./tensorboard/Models/'+folder+'/time_step.pickle', 'wb') as f:\n",
    "            pickle.dump(time_step, f, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluate on Test Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./tensorboard/Models/NEW_MNIST5_lr0.001/my_capsule_network\n",
      "Final test accuracy: 88.6200%  Loss: 0.123631     \n"
     ]
    }
   ],
   "source": [
    "n_iterations_test = len(mnist.test.images) // batch_size\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, checkpoint_path)\n",
    "\n",
    "    loss_tests = []\n",
    "    acc_tests = []\n",
    "    for iteration in range(1, n_iterations_test + 1):\n",
    "        X_batch, y_batch = mnist.test.images[(batch_size*(iteration-1)):(batch_size*iteration)], mnist.test.labels[(batch_size*(iteration-1)):(batch_size*iteration)]\n",
    "        loss_test, acc_test = sess.run(\n",
    "                [loss, accuracy],\n",
    "                feed_dict={X: X_batch.reshape([-1, 28, 28, 1]),\n",
    "                           y: y_batch})\n",
    "        loss_tests.append(loss_test)\n",
    "        acc_tests.append(acc_test)\n",
    "        print(\"\\rEvaluating the model: {}/{} ({:.1f}%)\".format(\n",
    "                  iteration, n_iterations_test,\n",
    "                  iteration * 100 / n_iterations_test),\n",
    "              end=\" \" * 10)\n",
    "    loss_test = np.mean(loss_tests)\n",
    "    acc_test = np.mean(acc_tests)\n",
    "    print(\"\\rFinal test accuracy: {:.4f}%  Loss: {:.6f}\".format(\n",
    "        acc_test * 100, loss_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Total Number of Trainable Weights in Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "633600"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum([np.prod(v.get_shape().as_list()) for v in tf.trainable_variables()])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Routing Path Visualization\n",
    "Visualize regions of input image that influence final prediction. Also can visualize the regions of input image that are routed to particular capsules."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Functions "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def visualize_convcaps(caps_and_routing, caps_and_routing_dims, reduce):\n",
    "\n",
    "    temp_dims = caps_and_routing_dims[0:3] + [1]*(len(caps_and_routing_dims)-3)\n",
    "    convcaps_lengths = tf.reshape(safe_norm(caps_and_routing[0], axis=-1), temp_dims)\n",
    "    temp_dims = [1]*(3) + caps_and_routing_dims[3:len(caps_and_routing_dims)]\n",
    "    convcaps_lengths_tiled = tf.tile(convcaps_lengths, temp_dims)\n",
    "\n",
    "    temp_dims = caps_and_routing_dims[0:4] + [1]*(len(caps_and_routing_dims)-4)\n",
    "    routing1_reshape = tf.reshape(caps_and_routing[1], temp_dims)\n",
    "    temp_dims = [1]*(4) + caps_and_routing_dims[4:len(caps_and_routing_dims)]\n",
    "    routing1_reshape_tiled = tf.tile(routing1_reshape, temp_dims)\n",
    "\n",
    "    temp_dims = [1, 1, 1, caps_and_routing_dims[3]] + [1]*(len(caps_and_routing_dims)-4)\n",
    "    caps1_lengths = tf.reshape(safe_norm(caps_and_routing[2], axis=-2),  temp_dims)\n",
    "    temp_dims = caps_and_routing_dims[0:3] + [1] + caps_and_routing_dims[4:len(caps_and_routing_dims)]\n",
    "    caps1_lengths_tiled = tf.tile(caps1_lengths, temp_dims)\n",
    "    \n",
    "    temp1 = tf.square(tf.multiply(routing1_reshape_tiled, caps1_lengths_tiled))\n",
    "\n",
    "    temp_matrices = []\n",
    "    index = 0\n",
    "    for i in range(4, len(caps_and_routing_dims)):\n",
    "\n",
    "        temp_dims = [1]*(i-1) + caps_and_routing_dims[i-1:i+1] + [1]*(len(caps_and_routing_dims)-i-1)\n",
    "        routing_reshape = tf.reshape(caps_and_routing[3+index], temp_dims)\n",
    "        temp_dims = caps_and_routing_dims[0:i-1] + [1]*(2) + caps_and_routing_dims[i+1:len(caps_and_routing_dims)]\n",
    "        routing_reshape_tiled = tf.tile(routing_reshape, temp_dims)\n",
    "\n",
    "        temp_dims = [1]*len(caps_and_routing_dims)\n",
    "        temp_dims[i] = caps_and_routing_dims[i]\n",
    "        caps_lengths = tf.reshape(safe_norm(caps_and_routing[3+index+1], axis=-2), temp_dims)\n",
    "        temp_dims = caps_and_routing_dims.copy()\n",
    "        temp_dims[i] = 1\n",
    "        caps_lengths_tiled = tf.tile(caps_lengths, temp_dims)\n",
    "\n",
    "        temp = tf.square(tf.multiply(routing_reshape_tiled, caps_lengths_tiled))\n",
    "        temp_matrices.append(temp)\n",
    "\n",
    "        index = index + 2\n",
    "        \n",
    "    all_paths = tf.multiply(convcaps_lengths_tiled, temp1)\n",
    "    for temp_matrice in temp_matrices:\n",
    "        all_paths = tf.multiply(all_paths, temp_matrice)\n",
    "\n",
    "\n",
    "    normalizing_factor = np.prod(caps_and_routing_dims[2:len(caps_and_routing_dims)])\n",
    "    all_paths_average = tf.reduce_sum(all_paths, axis=reduce)/normalizing_factor\n",
    "            \n",
    "    return all_paths_average"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Routing Path Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./tensorboard/Models/NEW_MNIST5_lr0.001/my_capsule_network\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABH4AAAI5CAYAAADXOP33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzs3WmcHUd9KOzTZ/aRZMna5UXybmOMV9msBgIOCBuzO+yEC8GEQEICmCSQvBBuCG+chTchYC434d4s/ILDeg04wrwk2AGMjYzxvsq25EWSZe3SSJqZc/p+IKnuas05s2jmzJme5/n0L1Wf7pJ6+nRNqf5VSZqmFQAAAADKpzrdDQAAAABgahj4AQAAACgpAz8AAAAAJWXgBwAAAKCkDPwAAAAAlJSBHwAAAICSMvADAAAAUFIGfgAAAABKqrOVF0uSJG3l9WaLNE2Twz3HL1cvc2+mwPfqX3Fv2pR7077cm/bl3rQv96Z9uTftazLuzZqFv+beTIG12//Wc9OmfKe1r2b3xowfAAAAgJIy8AMAAABQUgZ+AAAAAErKwA8AAABASRn4AQAAACgpAz8AAAAAJWXgBwAAAKCkDPwAAAAAlJSBHwAAAICSMvADAAAAUFKd090AYIokSeO6NG1dOziUewMApVffuy8qdxxzVIgffssx8cG5rsHQvLgvsOymelSef8um7Bpbt0V16eBQVqjG/Y1qX++obaZSqVQ7Gtel9UJZv42ZwYwfAAAAgJIy8AMAAABQUlK9YLYwFbV9jefeNEsTazU/UwAwZvUnnwpx50Cc6tU5kL1TD556IKrbdGFPVP7hZ66ZgtaN3XUDXSH+/St/LapbePf+qNx1+0Mhrg8MRHXVuXOmoHVToJjeldesX5ZM8xwLaWnkmPEDAAAAUFIGfgAAAABKysAPAAAAQElZ4wfKSh5vObmvADAjvee2n4f4FXN+HNW97NQLQ7z0b+N1corWfGj1xBrQEW9TnuTXp6nG8wHSWm3k4yqVStLdHeIlAz+Nr1FY1ybtjdcnalv5v+NkrY2T1kY/BlrEjB8AAACAkjLwAwAAAFBSMzLV63Wve12I3/Wud0V1TzzxRIgPHMi2QvzSl74U4s2bN4f4wQcfnIomAsTaaRt2AGDKdaxYHpVfMWddw2MPPvvUEPc+vD2qq8/ti8+7c2+Ia48+PvYG1eLUo7EmMBWPS4eHmxwcp0ml+5unrQGtYcYPAAAAQEkZ+AEAAAAoKQM/AAAAACU1I9f4ufLKK0N83HHHjekz7373u0O8Z8+eEN91112T1q6RPPbYYyHOt7tSqVTWrWuc5wuHrdmaMrYEb738v3nx3kRbiLo3AFAGtU2bo/LFF/1KiDe8cnFUl5yTxStv2RnXbd4anze/bk5hi/bqcceGOO2Kf9Wr7h2IG5jbwj2txn2TtD/bhr26bXd8/S1PZpc/5qj4c9vjttcHsmtW586pzAiFLekr+X+aui3amZnM+AEAAAAoKQM/AAAAACU1I1O98lu4n3nmmVHdPffcE+KnPe1pIT733HND/MIXvjDEz3rWs0L86KOPhvjYY7Npko0M57Yy3Lo1m4K5YsWKEY/fuHFjVJbqBQAAAEylGTnwAzSQWysmKeR8p/Xc2jFpm+UnF9e8KeM6N9XsfiTVxusvpbkB5WmTvx/FPHe57QAwJh3Ll0XlvScsCPFX3/3nUd3TuvtDfMmXL43qkoODUTnt7c3igcK6PU9uC2H95GOiququenzsUNbnSHfEa/M89t6zQ3zgnN6obvlXs/8gn7v2jrhtg3Fb47qhqJx0dzU8ttWSzqwtSWFtpEo9+3erH2iDflCTPmVay7WvjP1pJkyqFwAAAEBJzcgZP9///vdHjIvWrl074p8feeSRIT777Gw0+5Zbbgnx+eefP2o7Dhw4EOL7778/xPl0s4ULF4Z4/fr1o54TAAAAYLLMyIEfoIHclM62SBlqptl28+P53EyZxppLkUrrTY5rN+Np7Ey9NwAwBWpb4m3Y5/wk+0/j973zN6O6fcuzVKOFBx6KTzQv3ga9Pi9LC9t96ryorntP1t/o27AnPk8hhSndsy/EGz54dlQ3d2P2Dt+7L07JmvOtW7NzFPoJazfGa5iuWbk6O7aQBtZOqV7pUNa2dHioyZGjGGv/djx9pGq8fEM+vStaymGU8yadhfvf7r8rMKmkegEAAACU1Kyc8bNjx44Q//u///uIxzRLIRvJa1/72hDnU8nuuCNb8Ozqq68e1zkBAAAADocZPwAAAAAlNStn/ABtYKLrv1g3Zuq5NwBw2Kpz+uI/yK2p0nPLg1FVT5PzpAP74z94NFuPZsFjR8R1C7I1f9IntkRVh6zaV83mAKz6s5/FdaccF8K+bYVr5Nb1ueyOx6KqZ3/o16Py/Eq85s+McDj9manoC9XjLeSjZZWarSmUNNnqnVnHwM9hWLp0aYg/97nPhbia+xL9xCc+EeLt27e3pmEAAAAAFaleAAAAAKVlxg/Q3mwR3r7cGwCYcklH/H/1SV9vVhiKt0ivbN2WHVfYvr2+d1+lkeSMU6JyrT/ban3Ojx6Iz5OL3zl/c1T3lX9pnNqV9DRLaGPM8v2vZn2vpDDHoy7VazYz8HMY3vve94Z4yZIlIc7vGnbfffe1tE0AAAAA/0WqFwAAAEBJGfgBAAAAKCmpXuP03Oc+N8S/93u/N+Ixr3rVq0J85513TnmboNSsG9NexppXDgBMr/o43tPD8Wbvtd7s18Tqnj1R3dqN2To+a45/ZuFEhXVkcuvMFNccYgpUO7LYmj7kmPEDAAAAUFIGfgAAAABKyny7cbr44otD3NWVbXP4/e9/P8Q33nhjS9sEAAAAMBIDP0D7sY4MAMBhqQ8MNK0fft6ZIa71xIkgfT/bkJ2n2Ulqo6wjU02a1zOypMm/W5O+cZL7906b3jhmG6leAAAAACVlxs8Y9PX1hXjNmjUhHhwcDPHHPvaxEA8NDbWmYQAAAABNGPgBps5Yp6kWj5PeNfXcGwAonfrefSFOurubHrtldW+Ie3bG7/ee7+8IcX779kqlUlmzcnXjkyZxQkm1r7fBgTSVNEnMSXPpdfnt2yuVSjo8PEUNYqaT6gUAAABQUmb8jMEVV1wR4nPOOSfEa9euDfGPf/zjlrYJAAAAYDRm/AAAAACUlBk/wNSxHkz7cm8AYMZLa4337E5zG9FUKpXK4AvPisoL78vWg5nz4wejuonuBG5Nnwk6ZE3F3B0Y4/btxY9BnoGfBi655JIQ/+Ef/mGId+/eHeJPfOITLW0TAAAAwHhI9QIAAAAoKTN+gOmRn9Iq7ai9uDcA0L7q2bu5mM6VV3vuM6Jy331bovLQMYuyU+7aHdXlt3B/2cnPLZz5YBYWtx0vpB7RwCGpXePob+W2cLd9O2Nl4Cdn0aLsy++v//qvQ9zRkT1c1157bYh/8pOftKZhAAAAABMg1QsAAACgpAz8AAAAAJTUrE/1yqdxrV27NsTHH398iNevXx/i/A5fAAAArVQ/mFtjp1ZreFwyFO/tve8ZK6Jy73W3jul6af56BdU5fWM6B6M4nDV/YAzM+AEAAAAoKQM/AAAAACU161O9TjzxxBCfd955Ix7zgQ98IMT5tC9gHExhbV/uDQC0rXRwKConndmvcGkh1WvgknND3LMt3uq9WWpXfvv2SqVSWbNydeMGFbdwZ2yK/a28Zn2v3PbtlUqlUqk3Tu+DRjy1AAAAACVl4AcAAACgpAz8AAAAAJTUrFzjZ9WqVSG+7rrrRjzmiiuuCPG3v/3tKW8TlJ51Y9qXewMAbSsdHGxYVz1hVVTufSo7tt7V/P/41//DGSG++KITCrUPNb6mLdynXn49IGv6MAnM+AEAAAAoKQM/AAAAACU1K1O9Lr/88hCvXLlyxGOuv/76EKfSIAAAAIAZaFYO/AAtks9PNoDaXtwbAGhL9b37mh+QZEkbB49ZEFV17s3W+Om86c6mp3nghf87xGvetrrx5Tr9yjgh+b5WpTKu/lbS0ZF9bHh4slrELCbVCwAAAKCkZs3w7fOe97wQ/+Zv/uY0tgQAAACgNWbNwA/QAocxpZUp5t4AQCkkT8u2Xu/acSCue3BjiItv+rUb10XlSy64JFfa0vh6vT3jbiOV8fW1qh1RUXoXk02qFwAAAEBJzZoZPxdeeGGI586dO+Ix69evD/HevXunvE0AAAAAU8mMHwAAAICSmjUzfgAAAGa6oYX9Ie7++fqorr5//5jPU9ucW9cniecDVPt6J9Y4oC3N+oGf2267LcQvfvGLQ7x9+/bpaA4AAADApJHqBQAAAFBSs37GDzCK8WwDbovwyVf894/qCmP39VrjY90bAJgZOuKtvdc+fFPhgHWVRl78lneGuOuG26K6l53wrMLR2ZbhSXdXXFVt0v8gU9iGvZLWQ5h0jGOL9mZ9OJgEs2bg51Of+tSIMQAAAEBZSfUCAAAAKCkDPwAAAAAllaTWfQAAAAAoJTN+AAAAAErKwA8AAABASRn4AQAAACgpAz8AAAAAJWXgBwAAAKCkDPwAAAAAlJSBHwAAAICSMvADAAAAUFIGfgAAAABKysAPAAAAQEkZ+AEAAAAoKQM/AAAAACVl4AcAAACgpAz8AAAAAJSUgR8AAACAkjLwAwAAAFBSBn4AAAAASsrADwAAAEBJGfgBAAAAKCkDPwAAAAAlZeAHAAAAoKQM/AAAAACUlIEfAAAAgJIy8AMAAABQUgZ+AAAAAErKwA8AAABASXW28mJJkqStvN5skaZpcrjn+OXqZe7NFPhe/SvuTZuajHuzZsE7J+Xe1A8eDHF11TFRXbL/YFSuLZ4f4oNL+6K6vkd2ZoVtO+Jr7Nw15vZ0rFieXW/T5kJlR9bWnp4xn3M81u78O89Nm/Kd1r7cm/Y1GffmJd1vnJR7k9ZqIe6YN6/psUl/9o5J5/bHlduy9019YCC+xuDgmNtT7cuuUd+/v9CAai487H/CEV03+M+emzbVrt9pSWf863Najy+R5PpJSVfh2NyzkX8Wf/EH42hqkvunGc/nJkm73hua3xszfgAAAABKysAPAAAAQEm1NNULgDi1q1KpVDqWLglxun1nfOz+A/GHc6lXxUSr+qS0rpDelZuy/IvKwtRkANpWMZ0kSq0qvIuK3+/pnj1ZYcukN+0XbcindyWF/49O82+1wrsIWqma/fyNlqKV1rP6dGjsaY/jMg3pXcx8ZvwAAAAAlJSBHwAAAICSkuoF0AJRylYaJ2XlU6vqzzwjqqvedGd8ovxU+HSykrsaS552YlRO77x/yq8JwMQdkoqSk0+t6ly2NKob3vLklLVpLDqOmBuVa7t2T1NLoCCXvpV0dUdVU5bONUaH7DI2PDxNLaHdmfEDAAAAUFKzZsbPhz70oRD35Ra2O/PMM0P8ute9bsTPXnXVVSG+8cYbQ/yP//iPk9lEAAAAgEllxg8AAABASc2aGT8A06rJejzVefOyQnFNn3GcJ7/1eu2C06Oq7sd3xKfZsy/E9R1xXV4yUNjut9NrA2CmSjq7QjxZa/p0Ll8WldN9A3F5MFsD5ZAt5PPH1Qrvt+L27jBdkiSEk7WmT9LTE5XTocLaPPn+XpPt29O6rd0Zm1L34K+++uoQN0rjyqvXR/6F6t3vfneIL7roohBff/31Id64ceNEmggAAAAwZQylAwAAAJRUqWf8ALSN3JT1tRtuHvPHLj7nJVG5vm17iHe86fz44Nxs36E5cdXBC4+Kyqu++VRW2F3YMje/FfCuPfElctuEJt3xlqYwpXJT7UfVZFp8U9WO5vX1xttkQzuqnnlaXH4yS+1N5/ZHdemjT0TlfFpWx4nHxSfOPY+1I/qiqoFjV0blebduys5ZuEY+nSWfElasq1RGeTZhEnUsmB//QT6dqiP+Wazt3Bkfm3v/VHt7o6p8vylZvDD+XCHVq7Y5S8Vsml7WbAkAyCndwM9407vuvffeEH/3u98N8QknnBDiSy+9NMQnnnhiiN/85jeH+FOf+tT4GwsAAAAwhaR6AQAAAJSUgR8AAACAkprxqV6rV6+Oyq9+9atHPO6uu+4K8Ste8YoQP/VUts7F3r17Q9ydy8H8yU9+EuKzzjorxIsWLZpAi4FZKZeDfc4f/0ZUdesffC6r+5O4bumqeI2dJLfGz8JvxFu/73jF00M87/E453v5v2+PyvVHHs0KtXjdkgMvOSfEvd+/PaqzvS7Tprhuz3jW/Gkmf57iWgkTXSsI2kTyxFNROZ0/N8SDRy+I6rq7Cr8W3PNACKN3RqVSqZ6wKosH4vVH5v4o3iZ+OPfeKj5jnUdn688NP7G52HyYFvV9+6NytS9bqye/1mGlUqlUC9uy1w8cGDGuVCqVjt7s2HT7jqiuvndfVC5eJy+/FXyaW4sLmtGDBwAAACgpAz8AAAAAJTXjU71WrFgRlZPclO18etdLX/rSEG/atKkymg9+8IMhPv3000c85jvf+c6Y2wnMcrkUqbmb49Sq7wxkU4hv/cjnKs2869Hnhvjxty2L6hZcvS7EO18fp8H2zYu3FC2md+U9/pahEJ94ne2raRPF1K7cM5UUttctppOk+Z/3YvrWZKVzRSljUsRoD4NPPzYqb3p29i44/qUPR3V33xlvw778h88M8YK190R1tQceCnHHKSdWmmqy3fTBk7L3WMfjTzQ8DlqpOndO/AeLsrTI2rJ4q/eOXYV0rse3ZMfu2h3V1Xbuyq7R3z/x9h1xRHbOrVsnfB5mFzN+AAAAAErKwA8AAABASRn4AQAAACipGb/Gz7e+9a2ofNJJJ4V4z55sG+Tt2+OtjEfzhje8IcRdXV0TbB3Af8qtcdCzfSiqet/33hbiS175haanefTXjwtxdVNhe935Wc73grsL28DfF6/l0GwFkr5bmuSdF9dSgVY5ZG2e3Lo9hZ/LtD5F6/g0Y10f2lDPvY9H5WN3LQ7xy956Z1T3xMojovKC/z/bCr641XTSmesbb4u3pa5t3znm9nU9NZBdo1iZ+P9ppschP++DgyEeOmlxXFeLf3KT3O+flXphncTcWnBp7pyVSvPt24vSwjbxMBa+UQEAAABKysAPAAAAQEnN+FSvog0bNkz4s1dccUWITznllBGPuemmm0aMAQAAANpN6QZ+ANpRtT9bN6frJ3dHdaf9JIsv+aOXRnW1p7YVznRfiNLTTopq9p2Urc8w9/oHorp6IZc8OuNnzo3Kp/7mzQ2PrRTXToE2kA41/vmeMrm1Gn7RCM8G7SHJrXk1/ORTcWWu/O1zj46qlhy8PyrnVyfpWLAgvsaC7H1Te2xTfI30kNV6sqpnnxmV6zfe3vDY+DzWl6N1iu+UfLn7u+sKB8ff/flS0hn/qp0v1w8eHHN7OhYtjMq1beNbuxYqFaleAAAAAKU162f8vPzlLw/xJz7xiRB3d3eH+Mknnwzx7//+74d4YCDbiQAAAACg3cz6gR+AVktyA8tF9d17GtYV7TltflTufzzb3rO4FWkzC+4c+6sg6e0Z87FQalK7mAGSatKwrj44NPYTLYpTvSp7s//8TIfHfp7OHfF/mtYaHFepxClr0DbG8d1fTPWKtmwfx3nSA2NPC4NGpHoBAAAAlNSsn/GzevXqEHc3+F/4q6++OsTXX3/9lLcJAAAAYDKY8QMAAABQUrN+xg9AO0mHhpvWDz8v2wr3iH+Pt2yv1LLVEupNttOtVCqVJ79+YoiXv/G2uA35QhL//0BS3MIaZpP8z781fpjpRnlPdB61IsS1DY9P+DJDLz43xMkPbmtyJMx8SVeWQTKeLduLOpYtDXFty5NNjoSxmZUDP9/85jdD/JKXvGTEY/7hH/4hxH/wB38w5W0CAAAAmGxSvQAAAABKalbO+AFoJ/WBgYZ11f7+qLzt9N4Qr7gtTjVptoX7STfG2+JWX/ZU9rkmU5Grfb0N66D0iqmN0ruY4dJa4w3Uk86uqFxfPD8rPPlUVNdsC/cDLz8/Kvddl6V3Nb2+7dspgaQ7e44OeU6avEM6jzk6Kg8//sSktgtmzcDPihVZnvJznvOcEPf09IT4qaeyl9of//Efh3jv3r1T3DoAAACAySfVCwAAAKCkDPwAAAAAlNSsSfX62te+FuJFixaNeMw//dM/hXj9+vVT3iZgdmq2xkHRwIWnRuVlN+8OcX3PnsYfLGzDfsM/nxeVj9p5c+OP5lJgYVazpg+zSMfRy6NyunlbiOtN1vQp6n94d1SuDw42PNa6Psx0xT5Tml83cRzvkPqu+Lnx/mGymfEDAAAAUFIGfgAAAABKqtSpXq94xStCfO655454zA9+8IMQf+xjH5vqJgEAAAC0TKkHfgDaUZT/XVA99cT42GoSlZP7NmR1Ta5x2e0bo/JXnlFveGzSGb8KrLnArJXEz5s1Fpjpmq0p17FgQdPP1rdtH9M19r3mgqg85+uN15Arrj8HM061eR8pHR4e02k6Vx0blYc3PDrhJsFY+PYFAAAAKKnSzfjJ79j1kY98JMRdXV0jHv/zn/88xHv37p26hgEAAAC0WOkGfgBmsl2nx1Pv5z0YD0in+/c3/GzHksUh/uffODOq66zc3vBzSXf3eJoI5SW1i9lkUfy+SXcV3jdN0sSquS2s5/94Q1TXLNElKaQvw0xT7Y4nE4w1tatSqUTpxLUntkxWk2BMpHoBAAAAlFTpZvx88IMfDPH5558/4jHf/OY3Q2wnLwAAAKCszPgBAAAAKKnSzfgBmMnm370zKtfvf3jMn73nU0eH+JRfu635wbkt29PBwagqza1z0nRr9+LW1yfGW5MOLerPDh2Kt5Pfe2xviOc/uK95W2Eq5X+OrfHDbLItft/Udu5scOChhs9/WoirP/x5kyMr0RbuaX1iz1hxbaDqvHnxASuWZMcW1yqam72LKtt3Tej68F/qg0OFP2i8FlZRx6KFIa49tW2ymjRmSWf8q3+SW6srLfy9ktxaRsU6ZqbSDfx84AMfGPWY973vfSG2kxcAAABQVlK9AAAAAEqqdDN+AGaatRvXhfiS5xwVV6b1SiMdy5ZG5Yde8ndZYWN87Ete96tReffxfSHu2h9PvU9qWbneGU+vH+7NykmhaUlxBn+u3LUvPnjehmxb+uSOByowbZL8/4EVfqiLqV/SwpjhOk4+IcTp45vH/Llqf39U/u7V/6vhsZdccElUHj4qS2+pDhTSSYZyW2F3xqnF9f7urFB43urFVONcfTWdE19jd5ZOPLxte8N2QyPVOdnPVH3/gbF/sNnPaW9vVFVMIes4Ym72saHClvH556FwjaQr9+t9MbWykDIZ1XfHVflt6tOheEkAZqZZOfCzcGH2AhoaGl/O4q5dWW5w/rNdXVke5Pz580f87IIFC6LyWNLSarUsb/R3f/d3QzwwMDB6YwEAAIBZTaoXAAAAQEkZ+AEAAAAoqVmZ6nX77bdP+LNf+cpXQrxp06YQL1u2LMSvf/3rJ3z+ZjZvznKxP/nJT07JNYDple4Z+06D3/nptQ3rimssVDffFZUX3Nz4vElftv5PcevP+p49Y27fWFkphZaqdjSuG23dHuv6MNN15LZWH8cWzf/6wI8a1r3s5OdG5frAE/EBj2fl4qp1SUfueUzi/49Ohye2hfTYN9eGMarmfjabrL1Y1FFY5iOvfvBg/AeF90tt567KmDRZRwjyzPgBAAAAKCkDPwAAAAAlVbpUr2uvzVIfXvnKV076+S+77LJxHT+c2wqvXm88NfCaa64J8bp160Y85j/+4z/GdW1g5rn2jn+Lypecf3FUTg9k24iuWbk6qqvOPyLE9V1bJtyGfHrXVKR2VSqVyo43nh/im/70qim5BiVXnN6e12yqe70kiSDNUtaggeRAblvms0+N6qp3rY/KaW5n2ZcefU58ns5sN9t0+DB2ms2ld000tWs0nUetCPHBU5ZPyTUotySXIllM36rt3Nnwc7UdOwonyr23Dicla7LO0+wSPT0h7li6ZEquQWuZ8QMAAABQUgZ+AAAAAErKwA8AAABASZVujZ/XvOY1If7whz8c4q6urpEOjzz96U8P8Vi2ZP/iF78Y4kceeWTEY772ta+F+N577x31nMDss/oP3xPipTdui+oefePiqHzs3+a2Ze+I1/io7x77VvDNTHhdn0J7Krn1Iarz5kVVR/7zT7PCn07scsxys33L2nFsKQz/Jd2TvSeSp7ZHdbWzT47K1Ztz75tJ2mr9kPZM9DyF9uSfh/z6Q5VKpTL8xKYQb/2V4yZ2PWa1+r79Ic6vfVWpVCrVvr742IEma15N1ntrKt5/hXXz0vx28x3mipSBuwgAAABQUgZ+AAAAAEqqdKleeVdeeeWEP/umN71pElsCAAAA0HqlHvgBaEtnnxYVa31ZXvXWZy2K6u74wOei8pr/b3XD01b7+0N8SI55YT2EpLcnxOngYHxsIX+90Xlqz3l6VNUxMBy355HN2cfm9kd1j/z203KldY2vB43k1yMorndQWKuglOsBFdc4gRF0Ll8WlXc9d1WIj7hta1RX641/LUiavAuS3JpuxTVPmh5bLzyLY1yrqnPZ0vhjw/H7pp5fu6gz/ntUTz4uO8+LnhrT9Zjdkq7uqFxduCDE6a7d8bF9vfGHm63x02LFv0fxeYue3cI7pdqTrZVVnz9n0ttG6+k1AAAAAJSUgR8AAACAkpLqBdBiyd0PReWhi84O8QVv+3nTz2563wUhPvpLD8SV+a03C3a84byo3P+2J0Lc/Yfzo7rOzTtDnHbFr4mH37g8xMd+P57OnOyPt+V9+49uDvHHvvTmqO6oH2XpZT96azz1+MJDmw+Hapa+dchWz81TUWYk27kzBsNb4nSuuQ9n6cQHVh0Z1W09tycqH7vxuBDXH30iqmuWElxMy6rkUmHS3XuiqvRA7r1VLaQkr8jOk27fGdVVBuP3Te1Zp4e468m9UV2yfVeIF3z6mPg8l1TgEOlQnAKf7t0X4qSwfXsy/4ioXN2X9Y3qxX5Zk/dWMS0ryW2hXkxtPCRlMn/97ixFa7TPdeTanhaeqXQo+2x14+YKM58ZPwAAAAAlZeAHAAAAoKQM/AAAAACUlDV+AFqsuNXsMX+RbWf+6NePjepOecd74g8vzfKzk544HzxdMC/Ee89cEtXteWX6odBrAAAgAElEQVS8rsJNT/8/WeHr8SUu+Eh2zXpXXHfPr2fby7/k+2+Pr3/vg1H5i8/Itq0/vufOSiOfPPuFUXntzpGPgzGrT9GaPsVt4vOarjnUZOv5iSrjFvVMuqQa/8ymP7s7xD1z50Z1x2yN179J+7M1f/JbslcqlUqlO3v/dCyI14kbWrk4Kh9Ykh1b61oe1S346aZcY+O27j01W49ozo3xi6G2J36nVf/j9hDXq42f064btjWsg0bq+7I1fir7D0R11cF4PaBK8VmJDs7qis9UtbgtfE/2/FU74rkatR2NO0rJ3Gzr9XTnrriy8G6sFesbqO0YHP0g2p4ZPwAAAAAlZeAHAAAAoKSkegFMs2h67+Z4692T/mRrpZFDEj1y2+Qe8b2noqojvhcfenHl+Q3Pu7hyV8O6i6/OPtdZibelr/T3N/wclMJE06ukZdEm8ukl6f79ceVdD0zonLXCeaqb4q2fm70ZmiVl9j36eIjrhbpDUs+gVQrpUlEa2GjS7LNpMe1qaHLSqWrFLeThP5nxAwAAAFBSBn4AAAAASsrADwAAAEBJJam8cwAAAIBSMuMHAAAAoKQM/AAAAACUlIEfAAAAgJIy8AMAAABQUgZ+AAAAAErKwA8AAABASRn4AQAAACgpAz8AAAAAJWXgBwAAAKCkDPwAAAAAlJSBHwAAAICSMvADAAAAUFIGfgAAAABKysAPAAAAQEkZ+AEAAAAoKQM/AAAAACVl4AcAAACgpAz8AAAAAJSUgR8AAACAkjLwAwAAAFBSBn4AAAAASsrADwAAAEBJGfgBAAAAKCkDPwAAAAAlZeAHAAAAoKQM/AAAAACUVGcrL5YkSdrK680WaZomh3uOX65eVo57kzT5p0hb/1f8Xv0rh31v1iy6fMbcm3T//qhcXbwoxL97w3eiuuf3ZvGaV7wlqqv1dYW44yd3xeecO+dwm1mpVCqVtdu+4Ln5L8XnJsn9n0C91tq2VCbnuSnLvUl6eqJyNVeu7d7d6ua4N83kn6MZ+r4pzb2pdkTFpCMrp0ODrW6Ne5NTnRO/w5Pe3Hfa9h2NPzhFz5R7k0m6uuNyd9YXq+/b1+rmuDc51f7+qJz0ZZ1oz800m0F9aDN+AAAAAErKwA8AAABASRn4AQAAACiplq7xA1OumMvabM0fJl06PByVa5u3hPh9t78xqrv9gn/OCj+/N6qLV2eAWawW54entdbni8OMk9YLf+Ct0jaK32GduV9FpmFtLHIOeW5oF8V3f1LPPSueG8bIjB8AAACAkjLwAwAAAFBSUr0oN9Mf20ZXRzxN9dx1rw/x0sr6hp+rzps3ZW2iAdO921Z68OB0N4FGvG/aVjo8NN1NoIF01+7pbgINeN+0r/rAwHQ3gUbauA9txg8AAABASRn4AQAAACgpAz8AAAAAJWWNH2DCRssxrh6/MsTLPhyvf5FufDzEyaKF8QcHc+sxLCnUbd0+vkYyOmuTtK10eHi6mwAzj++0tlUvrhsz1nuVJBP7HGPmfdO+rLfUxmbQd5GBn3E65ZRTQnzvvfeG+P3vf3+IP/OZz7S0TQAAAAAjkeoFAAAAUFJm/DB7mCY8+Wq1ptXDS7Kt2B/7pTlR3co/fTjE//rAj6K6NStXZ4W9+6K66tz4PEwxz820Sjrj13R+Kn7S1R3XDQ22pE3Q9pp9b/lOa73cv3m1ry+qyqeMV+fE7/f6vtz7332aetWOuFzP+nhJT09UJfWotZq976v9/VGdrd5pxMDPOJ1zzjkhrtfrIX7sscemozkAAAAADUn1AgAAACgpAz8AAAAAJSXVa5zOPvvsEO/L5R5/4xvfmI7mMB7yw1tuz6osl/+IDfWGx/3oQOM6ppnnpvVy62Gk9Sb//qnnBkbke6u9JNn/M6dDTbYMH2XdQKZAfs2rZu8U96b18n2BJv/+zeogz4wfAAAAgJIy8AMAAABQUlK9xuCMM84I8fve974Q/+M//uN0NAemVbMtPDsWL4rK+5dkY8sr/u62qO6YH2Vbg/73pz27cKZs2qrt25l1cmkR+e10K5V4e/f81u5AE1E6izSwVku6ct9bhT5Efpvw+oEDLWsT/8n7pm0lHR0hLv7757d3b9YvhzwzfgAAAABKysAPAAAAQEkZ+AEAAAAoKWv8jMFpp50W4jlzsvVGrr766uloDuMhr3/SpYODDet2vOD4qNy1N/s3L+Yg/89j14X44iN+Oaqr79hxOE3kcHluWiv/7z3qsf6/Bmhzxe+0euP3SLSOjLVKWq/ZFu65NWYq1viZeoXnJm323OTXzRpq3C+HPD1IAAAAgJIy8AMAAABQUlK9xuDDH/5wiDds2BDidevWjXQ4AAAAQFsw8EO5WZ+kpXacFk8iXPGjseXrpwMD8R/k88phtmm25gJAuyusRZbWao2PbVbH5BvPmnJN1pihBZr1Ber6CYyfVC8AAACAkjLjp4HjjjsuxKtXrw7x/fffH+J9+/a1skkAAAAA42LgB2iqvrfxAGfH8mVReXBBPPW0e1fjLSbPuvI3Qrzi4M1RXXXunPE0kckmRbK1ilu01xunPqTDQ1PcGJghiikrzb63fKe1VH6r6UqlUkkHG/cFUtuEt5b3TdtKCsscNEuRrA+6N4yfVC8AAACAkjLjp4EXvOAFI/751q1bW9wSAAAAgIkx4wcAAACgpMz4AZprsp3k1otWRuUF9xYOuPWeED788fOjquM/fnOloWIOOpTJeLbTrcY5/83WY4BZxbo97eOQ76nCvcndq6SzsP6PNX5aq9kW4d43rTVaXyD/HefeMAkM/DTwjGc8Y8Q/v/LKK1vcEgAAAICJ8d/qAAAAACVlxg/lMp7tXTlscx+Lt2itdzUeS77vHVdF5TUfXz0lbWICPDetdci/b+Op94dsi3zQ9G4Yle+01iqkD6VNvqaSnp74WKlebaPa3RWV6we8b1qp2fbt1d74uakPDEx1cyghAz85z3rWs0L83/7bfwvxrbfeGuLvfe97LW0TAAAAwERJ9QIAAAAoKQM/AAAAACUl1SvnoosuCvHChQtDvHbt2hAfOHCgpW1inOTxT4pmOffVI48Mca033l5y7h2bonI+W/kzO1Y1vqDt26eX56a1xrP+SJOcf4C2UHyHN9lq2po+LTaO9417M82a3Zsh94bD57ctAAAAgJIy8AMAAABQUlK9cs4666wQp7npdl/96lenozkAAAAAh8XAD3CI9MDBhnUPXHFSiBfcE9f1PR6v8fPAn68O8XfOWtfwnNU5feNsIcww+XUWiuthpIX1MKrZ2lnWXIAJsG7Z1Mt9TyUd8Xp/aWGNn6Qz+3UjPdi4f8EkKa7r04z3TWuNoy8QPTdDg1PZKmYJqV4AAAAAJTXrZ/wsX748xBdeeGGI77vvvhB/4xvfaGmbAAAAACbDrB/4oQTy0yZN754cab1hVf/j2b93347m04LX/8rnQ7zmigsaX8N27q3nuZk+TZ6vSqVSSbryaRG2c4cx8Z02fUb7TuvpyQ6VTtRWvG/aV9LdHWLPDZPBb1sAAAAAJTXrZ/y8/e1vD/HSpUtD/K//+q/T0BoAAACAyWPGDwAAAEBJzfoZP5SAXP6pVVh/pzqUxXPv2R7VNcvyTzri8xR3sIZZY7TvrLrvNKC9JdVsTaW0NsoLfbR6po73TXvJ96nrzZ+LUZ8rGKdZP/CzatWqEf98x44dLW4JAAAAwOSS6gUAAABQUrN+xg9QqdT37mtYN3DJ2VG5Z1eW0FVf/0jT8774Le8McXf1nqiu2tc3jhYy6aRITq381tLjlA4NTmJDYJbwnTa1mn2njfJvXz94cJIbw2Txvplih9MXGHRvmFyzfuDn5S9/+Yh//q1vfavFLQEAAACYXFK9AAAAAErKwA8AAABASc3KVK/nPe95IV6+fPk0tgTaRNp4I/bHXzsUlY/9cuOvjUevuCA+9s/XZZcoHJv09Iy9fVBmxTUArFUCtJsk/r/itMk24Eln3E9Ih4enpEn8p/GsI+N901qF56ZZf7tS7YjLo2z3DuNlxg8AAABASRn4AQAAACipWZnq9epXvzrEHR3ZtLpbb701xDfccENL2wTtqr4//pro2t14yvan3/k/o/Jf/tlZWaE43RXKbBzTu5OOeHq3tAgYAykrLZVU439vqV5trMmz4H3TvpKuwnNzUKoXk8tvYgAAAAAlZeAHAAAAoKQM/AAAAACU1KxZ46e/vz/EF1988YjHfPWrXw1xrSavklkslwN+4j/Hz0LXbetDXFy1ZCBtvEV70mGcmVmkuKaP9UeAGeyQNX2abDWd+r6DX9AXoI34TQwAAACgpAz8AAAAAJTUrEn1GhoaCvGOHTtCfM0114T4r/7qr1raJgAAAICpNGsGfmhDSdK4Tg5sS1VPWBWVDx67IMS9D2+L6tKjl2WFBzdEdZ8//fTCmbPc5rQW5zk3ufs0k39uksKkzXwuuWeo9arZ2lhJtfATnrtX6dBgVJUOD09ps6CdJZ2Nu6Jpfr3F4nea77jJV+iXVefOzVXFdfnvrfrAQFx38OAUNI68jsWLskLhWajv3ZdVFe6F983U61gwPysU1saq5+7HIffGczP1ZvnvnlK9AAAAAEpq1sz4yad6Pec5z5nGlgAAAAC0xqwZ+KENNZtSl5+KNwum3k23+kNxytbwKdkU4v0nLIrq/u0f/i7ElzznFVFd7bEnGl6jOnfO4TSR/xI9D/WGhzEN8ql2SVdUlXRn5WKqF8xmUepJcRp+Pp01bbx9OJOk0N9K9+/PCt3dUV3+O60SZ3rRArXtO0OcdHTElcVUY1oqn2pXTMlPurJfvf12Mw1m+e+eUr0AAAAASsrADwAAAEBJGfgBAAAAKClr/DB9mm1LnSevf/Kl8dowtec+IypvOT/7arj3XV9oeJo9/yPOK+9/eZPtxZkcuefmkLz+HFu2tkBx6+O+vqyqvy8+Npcvnthed3rN8u1c2041+x7Lr39RlA4W3ifu1eSrxu+U6rx5WWHRgqguyW1TXR2Kv8Pq+/ZVmGSF7638+786p/C+yd2bei3uQ3vfTIFiX6C/P6ua0x8fm/veSgvPjfX/WmCW96HN+AEAAAAoKQM/AAAAACUl1Yvpk5+mXZx5L0VoahVS67rujLdzX7T05DGdZu6bdkXleuG+2cJ9CuSnCdelOkyrQqpJPbf1cUd3vJ17WsuejbJOIZ6xpAxNr3ouFaVemHqff6e4T1OvHqcF1XbtDnHnEXPjY3PfaVK7WqDw858OD2WFem9cl0vv8r5pgcK9qe3ZE+LOQqpXGqV6Se1quVnehzbjBwAAAKCkDPwAAAAAlJSBHwAAAICSSlI50wAAAAClZMYPAAAAQEkZ+AEAAAAoKQM/AAAAACVl4AcAAACgpAz8AAAAAJSUgR8AAACAkjLwAwAAAFBSBn4AAAAASsrADwAAAEBJGfgBAAAAKCkDPwAAAAAlZeAHAAAAoKQM/AAAAACUlIEfAAAAgJIy8AMAAABQUgZ+AAAAAErKwA8AAABASRn4AQAAACgpAz8AAAAAJWXgBwAAAKCkDPwAAAAAlJSBHwAAAICSMvADAAAAUFIGfgAAAABKysAPAAAAQEl1tvJiSZKkrbzebJGmaXK451iz8Ndaf2/q2SXTwcGoasdrzw7xub/986juc0f/JCoff83lIe6afzCqO/Hyh7LC0FBUl/T2jK+9E7B2+98e9r355eplbfvcJJ2dTcv1g/H9iKTT+9f6Xv0rpb43M5l7077cm/bl3rSvybg3axa8c1rvTfF9Xj1hZYjv+eD8qO5//dIXo/IffuhdIe7bciCq67jtwRCnw8PxNXpa0E/b+Xflfm6qHVEx6YjL6XCubzzN/bIi32ntq/T3Jon/eoc+N/F3VTtpdm/M+AEAAAAoKQM/AAAAACVl4AcAAACgpFq6xg/k5df1KeZKLrh6XYhvGzwvqnv+r62Iyqd98K7sPIUc9GbJo0ll6nPHyy6t1eI/KOTAtlu+OAAwNtG6PoX3ff2Bh0O88KfnR3UvvLgeleded2eIx9NPYxKk9UK5sPyHfhocqvBcpPVyPCdm/AAAAACUlIEfAAAAgJKS6kXLpLV4umnaZHppx+JFIe48EB+39+vLo3L/wUcnoXVMimLqFwAwIxySvt0kvaF65JEh7t4TH3f8Ny+Pyqce/NnhN46JKWnKCrRUMWVyhjLjBwAAAKCkDPwAAAAAlJSBHwAAAICSssYPU6a+b39UTk4/MSpXN24O8aY3PS2q23lOttX76X+wIarrX3xkVG6arZzbXrw6d078uQPZlqLF7UWLx5KT5LYCLeaODw+3uDEAwETUBwaicsfRK6JyumNXiPc9/7SobmBp1r9a8q0Ho7oFdy2Oz9OsEUn2f9DVOX3x5waHcvFgVFft7292VhqpW4sRxq3JurQzyawf+JkzJ/sF/8/+7M9C/O53vzvEt9xyS4gvu+yyEG/YEA9IAAAAALQTqV4AAAAAJTXrZ/wwhQpb36X3PhSVh57z9BDvOjWeevq033s4K3R1Fc4TTyluZverzwnxk5fG6VzH/kM2Tbn3h/dEdfW9+6LyrE79yqd2VSqTN92xeN6puAYAMCa1J7bEf3DmySE8sLAjqlryf+7LCtW4Lr3rgTFfc+2Gm0N8+lW/EdUdd3WuPY9tiuqKaWpSv6ZAk9R+YOaZ9QM/K1Zk+czvete7QlyvZ4MW5513Xohf/vKXh/izn/3sFLcOAAAAYOKkegEAAACUlIEfAAAAgJKalaleS5YsCfHf//3fT2NLyie/RXpRcurxUbn71vUhPvWmofjgeXOzuLdn7A1I4rHMzgNZyt5Jb78rPja3BtFjH7ggqjr60+vGfs2ya0Vet9xxAJhy9YON+2nV446J/+CBR0O48J7huK4/23o9KWzDXtmxo3EDCv20E76e7aJ7yv8b973q+bUin3FqfJ47x76OEBOkbwalYsYPAAAAQEkZ+AEAAAAoqVmT6vVbv/VbIX7Vq14V4gsuuGCkwxt6/vOfH+JqNRs3u+2220J8ww03TKSJM1Jx2/NIYTrv5v8eVx/13lw6V2GacDKYpX6lu/fEdaedFJWHlmRbeHb+x+1RXf+3f9a4fTkrr340KtcKW9Hn/56l3Np9qrZsb8YUYgCYUsVtz5vZ9JLlUfmor2V9n6Svt+HnRuunpX1dWeHWe6K6U377ljG1rfrE1qhcL/bTcn9PW7sDHMqMHwAAAICSMvADAAAAUFIGfgAAAABKatas8fPpT386xPV6vcmRzb3mNa8ZMd6wYUOIX//614f4llvGlrs8k6S59XeSzvhHKB3Otvt88FPnRnUnfjJeD6i+6Iis8MCGqK5y7FEhHD5tZfy57ni8snqglhU6OuLz1HPryKSN73vt8U1RefdrV0flI3+UrQFU27wlvv5MXPNnOtb0AQCmXDo4GOJm/bSdb4j7Ost/GG/Dnh6RrcWYbnw8qkuOPzbE+54er+nTke+XVSqV3k17Q1yfYD+tvm17VK6eemJ8wJNZfb2wnbw1fwDM+AEAAAAoLQM/AAAAACVV6lSva6+9NsT5rdfHa9u2bSHeuzebrrpq1aoQH3/88SG++eabQ9xRnNIKAAAA0CKlHvhhaqS1XO52Lc7jzq+xM//+eB2Zg0t6o3Lvd28NcXLGKVFdrTs7T/WmO6O6rmOPjsoP/MmCEL/ghPhH+olXzQtxffeeqC49eLDSyI8//fmofNGb3xHizsIaP/X9B7K29sV/x7bVqjV98msJJYXB13rhZ2ci56xUrE8EADlp/r3YpJ/Wuy2uq83pjsrVux4Icf2Cp0d1w3O7Qty39mfxJZYvi8r7zsrWbdz5ovOiuqP+5cGs3XvjtSCb9dMOLpsblXty609WCmv81HPnqfb0NDznrKd/BeM3g54bqV4AAAAAJVW6GT8veMELQnzqqaeGOL+T11h29fr857MZH9ddd12Id+3aFeIXvehFIf7oRz864nne8573hPiqq64a9boAAAAAk6V0Az9Mr92vOifEy/7l7qiuXpjCW120MMTJlnibznTrUw2vkXZ3ReXen2bbqV/5vO9FdS98/YdCvPj2eMpw1w23NbzGKdf/alReNZQbLDwkZal9p/RF8lMR22EaYru1BwBKbudlWT9t4TWFftrAQFTO99M6Ho7T3KtN+mmVQj+ta3e2hfz8h+O0iIHV2XqZvVv2x+f5+b0NL9E5MBT/QbN+xEzppwFMIaleAAAAACU142f8HHfccVH5y1/+cogXL1486uc3bNgQ4q997Wsh/qM/+qMQDxT+B2Skz15++eUhXrJkSYivvPLKEPf2xgv//s3f/E2Ih4YK/3MBAAAAcJjM+AEAAAAoqRk/44f2sulF2dag86+NF9HuWLY0Kte2PNnwPPd9NstBX3B7nCt+60c/F5VfduqFIX7LN94Y1R21Kdti9ODzz4iv/+xnZG278Y6o7oS33hWV83nuT70p3op04Zd+euhfoB3l1yZKJ7iVeqXSfG2eZlsaHs41G50TAIg1WdNm2xnZe/rIb8Tv5Y4l8Uz5WpN1fA5elPXTOvfGs9a3Pr0vKi/9p9tD3H3kgripT20L8fDq06K6zjNOCXF65/1RXbIuXp+oMv+IrO60k6Kq9N4HKzPCdK99qH/FTOS5GbMZP/DT2Rn/FcaS3nX99deH+A1veEOIn3qqyUJ1I8inen3qU58K8V/+5V+GuL+/P8T5tK9KpVK55pprQrx+/fpxXRsAAABgNFK9AAAAAEpqxs/4YRrUGqfsPHzp/wzxC695V1TX+91bG35u7cZ1hT/Jyg+v2RvVrDn+xfGh9Wyb9nT3nqgqrWXpZv/2v/82qnvGTW8K8coPHB3V1R59PL7Etmy7+dd+6OGo7gdXL6qUWjF9q5kJTndMurrj0+R/xurjSBFrlmoGALNBWm9Y1XFC1qc6+Ow4tar7B7c1/Ny2t50flfccl8VHrI9/nVjyxVvi5uTak+6LN0zJ99P++p/iVP53fOQDIV64bXlUV9u0OSrXd+3O6p62MqorZgcwAdWOuJz/GdPXgpG12XMza74J163LBhLe8Y53hHi86V2N5NO23vzmN4f4/PPPH+lwAAAAgCkn1QsAAACgpAz8AAAAAJRU6VK9qtWRx7Ke+cxnTul1k9zaIvk2NGpPpVKpfPzjHw/xW9/61ilp13QaWBL/ePUW6vPr+lxywSVRXbSFaGFNoaQ7zpdMK1l9Psf7F8dma8c8NhyvFXTgvvm5wvbKWP376oVROensaHBke0mq2c9ok/T/X8jnpI5njZ3pJs8cABpatWhHiPcsOiaq6y4cu/F3Lwjx8X8X7z67aMfOrFDspxXW1Enzy/btiddizPfT3nLH2+O6vly/ZXCwMlYdP72n8Aczo582LtO9hTUwulF/4WotM34AAAAASsrADwAAAEBJzfhUr1//9V+PyvX69EypuvTSS0N8zjnnhDjfnmLb8qleAAAAAJNtxg/80F6edUU2ELf4ugejumZDcvV8rnilEuWL5/O/K5VKJR0ajo894+QQVh97Mqo64pvZeV770SuiupPXZu2r7dhRaeb+T58X4lN+55a4snNmPEZprc3W6smtI5QOD8V148lXl+cOAGMy8DdHh3jBfzwQ1RX7aUnuD+p74nUSm/bTCuvxdBy9Iqvbuy+qW/87p4Z46RfjFsy74b7s+oU1HIsGX3R2iLv/7edx5Qzpp42rD9OK/k6+fzWT1ntkdmnnvn+btU2qFwAAAEBJzZAh8MbyKVatsGTJkhCffvrpIf7IRz4y6me3bt0alYeGhhocCQAAAHD4ZvzAD9Nr6AVnReX+LdlgVn2U9Km8J798dFRe9tYtIU4PHIzqqietij/8yBMh3Pv8k+O6d2UpZIuevD+qqu/cNeb2PfTa/xHiNb+zOqpLD2btS7ra+JEa1xTi3HTr/FTf8Z6nmcmaNpzkJi6mpiIDwH8ZvvDMqNy3+UCIR+sHded2Xq+deVJU13nXw9l5Dsb9tI7ly6JyuiO7zvAzTojqTvripuy4QjrXaOldeU+ek6WbHfNvcV3UT+vvH/M5Z702S1MBDo9ULwAAAICSauPpCe3pox/9aIjf+973jnr8I488EuJf/dVfjeo2btw4ae0CAAAAKDLjBwAAAKCkzPjhsFSH4603d57QG+JFo3x2zcpsrZwlZ8dbge7/yoIQ7/nyUVHdki/fHpWT3p4Q96+9LapLc1uMpvv3j9KizNqN6xq2tSjp6WlY11bGs+15vr64xk9uG/ZoLaCxnHcyFNtji1EAGFHnznj9nYFVc0I82mo3yz5/c4iL/aLj/8/lIT7me/F7ee7aO6Jy0t0V4urNd0d19VxdWlgrqJnNv3FBVD7mL25ucOSh282XTrFflGedHhib8fyeNEMZ+BmDa6+9NsSnnnrquD57993ZC+6HP/zhpLUJAAAAYDRSvQAAAABKyowfDksyHE+FW/zVO7NCX198cC1OyXnyrdlW8L3b45Sh/k9kU38X3/jTqO6+vzovKp/2/zwQ4upxx0Z16RNbKlNupkwHHEc7k87sqyGtFz7XLLUqnwY22rETlRTGq23hDgAjqs3pispzvn9PViimqhf6Cf/64I8bnvfUL+bS52+9J6o7pJ/2B/eHuOOYFVFd/cmnGl6jmaE5ox9TasX+Vl6zvlcxLWym9GFhMkw0LbIkz82MH/hJCjeiWh15EtPLXvayEf/8C1/4QoiPOuqoEY/Jn7Ner494TCOXXnrpuI4HAAAAmCxSvQAAAABKysAPAAAAQEnN+FSvq666KipfeeWVIx737W9/O8SN0rXGksY1lmM+//nPj3pMWQysiPPDj9iR5W7XH3gkPriw9Xf3niw/stYdp+x13BhvBZp3ym/fGv/B/HnZNR95ND7PwiOzazTZzn0827cX15jJb1PaVg4jH/WQdX3GairW9GnFukEAUEK7T4jXW1y0bXmI6w9uiA9Ox9pxBu8AAB1bSURBVLGcQWFdn7xT3n9L/Afzjwhh7YnNUVXH4kVZ3ZYnG55z4+/G27ev/NPG27cf0k/rbNNfd6Zj3ZCpuEZJ1j+BhkryM23GDwAAAEBJGfgBAAAAKKk2nfs4dl//+tej8hVXXBHiJUuWTPr1tm7dGuJ77smmuV5++eUh3rRp06Rft10dcf1DUbl+XDaF+MG/Pz2qO+ltd0blhTdkaVn1XbujurQjS++p9vXGFy1uP7r/QHbsyqPjuoEDlbG45HmvKvzJYw2Prc7pa1jXVg5nW8J2Sqdqp7YAwAyy6Nr7o3L9uCwlf/N74rT25Z+L06cuee4rs88Vt13vGA5htdgvK/Tb0oEs1b56wsr42D0DIze84Pi/j9PSmvUMDuk3tqvDSR9pp75RSdJgmAVm+c+qGT8AAAAAJWXgBwAAAKCkDPwAAAAAlNSMX+Nnw4Y45/cNb3hDiF/1qmzdlve///2Tcr1PfvKTIf7sZz87Keecaarzctun79gR1w0NhvjBF14f1b3s9DdE5do98fpAeUlHNiaZzJkT1aXz58bH7t6XnfPhjQ3P2bFieVT+zk3fztp2UnehAfGY6IxZ16cs8msQzfJ8XAAYj+rcrN9U37krrrt/KMTDHzk2qkv+7aSoXLv3wVxlYYv0fD/tiHlRXW3Zgqjc8WTWhtoDDzdsd35r90qlUum4OusLDL10W8PPVSqVSrW/v2n9jGOLdGCSmfEDAAAAUFIGfgAAAABKasanehXdcMMNI8bXXXddiPNbr1966aUhvuaaa0L8hS98IcRJbrrl3XffPXmNBQAAAJhCpRv4YerVBwZCvPdV50V11eEsB/mS8+O1efafFeeA963vCnE6OBTVpcPDIa5teTJuQLHcRO1ZZ4R47b/87zF/rpLWx35su2qWH95uueLFtgIAE1LffyDEO1+/Oqrr3pf1b1a9N14Xcei4ZVG5o6cnxE37aZu3xA0olGtN2lo98bgQP/InPVHdwcey9RdPru5ucpYSard+GjDjSfUCAAAAKKlZM+Nn7dq1I8YAAAAAZTVrBn6YPElHR4jnPRBPvX3oI1n61uMvWBnVHfeteJpw0pX9+KUHD0Z1HcuWhviQVK/c9SuVePvP7/z02kJr14VozfHPjKtq2eTjpLCbeym2BS1OE67m/t2KqWytmFLcLJ3LlGYAmBT5ftqRd+yM6o68KkvDuvH2k6O6E74SJ2V15s6TpoV+2pLFIa5tfarQgDihoLow2979vj8/OqqrD2R9wVPfdHt8nlw/rdIZ/8qS9PVVSkc/CcYv/9x4TpqS6gUAAABQUgZ+AAAAAErKwA8AAABASVnjh3FLurN1fOp3r4/qTvrgkhBve0G8Ts6GS7qicteFTw/xcF8hJ3Pl/hAu/la8VtCNf/H5Mbd1zaoLskIa565X586pzCr1JhuqNskrTzrj+5Zfm6k+MND8PEmTseVm7QEAJiTJrYdTv+ehqG7Xr2Rr8yx7YfyOfvjV8Tu85/lnhPjA0uGobu6KvSFe+pl43Z7HfileOPG+d14V4is2nxPV3fHMXB+jsP5gKdZbHI9m65MU+1f5Y6vx2peR0fpa1hVippvoz+ks/Nk34wcAAACgpAz8AAAAAJSUVC8OS3VOvJ1mujeb+rvgX34W1S386bFR+YmXLgvxq9/xo6juT5ZlW3p+6bxFUd3Fv/S6qHxgVbZNaN8DWwst3BSipLuwZzuZ4pTGaGvEeOp1mt9etTi9uDilOJXOBQDTpdrXG5XTPU36aTfFKVuPv3xFiO+5/AsNr3H/ufui8m9f9Nao/OIfvDPEvesL/bT0iRDqpzUxjtSTpJr14dK0SYrYOM8LpRYtTxH/7lOW58SMHwAAAICSMvADAAAAUFIGfgAAAABKKklLkrMGAAAAQMyMHwAAAICSMvADAAAAUFIGfgAAAABKysAPAAAAQEkZ+AEAAAAoKQM/AAAAACVl4AcAAACgpAz8AAAAAJSUgR8AAACAkjLwAwAAAFBSBn4AAAAASsrADwD/t727i7Hjug8DPnOXuyR3+S1KpL4s6supFVpWlcCR28KyilqmKQt+sCQYVQvYbuTGjh8KP+TFShTAjoIIrVC0hZWiRWG3TZgqhuwisaPAqCQYBhpLlCDbqqlYH6VMKaJMWuLnLsndvZOHIDPnnMsZ7t7de3d39vd7OmfP3DuzO/fMzD17/v8DAAC0lIEfAAAAgJYy8AMAAADQUgZ+AAAAAFrKwA8AAABASxn4AQAAAGgpAz8AAAAALWXgBwAAAKClDPwAAAAAtJSBHwAAAICWMvADAAAA0FIGfgAAAABaysAPAAAAQEsZ+AEAAABoKQM/AAAAAC21Zpg7y/O8GOb+VouiKPKFvseHR+5xbgbgu7OPLvzcdO52bgbgu90/dW6WKedm+VqMc7Pn4n/t3AzA40f+s36zTLmmLV/OzfLl3Cxfzs3y1XRuzPgBAAAAaCkDPwAAAAAtZeAHAAAAoKWGmuOHVaZIQjfzOORwzc4dQSX+KM4e/nn1NtPnGt9nzsfQGYnfZqSq9+yjyXz23wbp75ueV+DvNF0b9JuB6p46HdU7V14W1b/zvW/WvvZDv35fWV7/xI+jtnxifO7HcOJUtf+rr4za3vrQJWX5kv/5w6itmJ2N9xncD/N1a+e8f1hs+drq85cn17fu2bNVZVDXt3CfrqGsEPrN8pWPjtW2FTPTQaWd58aMHwAAAICWMvADAAAA0FJCvRieZEpb9/iJstzZuiXedD6hV3PefzeqdjZtro4lCRPIkqn3xczM4h8P0C7hNW61hYQutW5yf3ntjTm/dN13q9CrdOJ132cxud89+7uPlOVbJn8jatvy/07GL33hpWr/Qr1YQsV09ezTE/Z45szA95+vGa2OJXkuS5/phLSwXET9Zmw0aRz85zTqN2H40pD2v5yFf498LAn7Gsa5CdOM9FzTBr9/M34AAAAAWsrADwAAAEBLGfgBAAAAaCk5flgWZt74m6geLmfbEwPZryR2spiq4tPDmMssS5ZbvMD7tD6XxyqPB4a+6DfL1p6r3h//4Mbry2Ln4JuLso/uwddr207siv/ntuV/vVSzZdaTuyjrtPx+w7KSB5+37smTSeMQliUO8vjkyWe/mHGNZfnrDiEXVir63pR30sbhHswyVjR91xvUPhfrO22fzPgBAAAAaCkDPwAAAAAtJdSLxdUw3TcM38qyZInDyeRtBjAVrrNuXfyDbjWFeCmmYgLQv+6JU7Vts//ol5OfPF2W8tH4XtT5RRDCktwnij7vDT/7rV+N6h8+sKMsv+sP9s/5fYqZmajeszQwLKY0dD0NEwkNIryrE4fdF2GoY1eICstU0m+iJbuH8blN+k1Ev4mF17RhhL01pQNZgpQAq2bg5+abby7Ljz32WFnetWvXou/r9ttvL8sHDhwoy4cOHVr0fQEAAADUEeoFAAAA0FIGfgAAAABaatWEen3kIx8py2vXrh3ovu68886y/JnPfKYsf/KTnxzofpfEPOITO5s3JT+oj0lds+OSsjxz+K24sc/l03uWaG+KXU8NY9lSAGoVp+NkcJ31VT6e7lSci+fQb8a5cf7h732+LO+YjnPsHP/D6lFow52H431s2tDXsV79338W1Ys/rZ47usm24e+RZVmWb9pYbXv8RF/7h37ka5IcUp365618dKwsF9PnFucA5CNhBQpz+mRZlmVFepUPttVvhqsp/1EizEWb5tfr2zL7zmjGDwAAAEBLGfgBAAAAaKlWh3qtCaZs7d27d2j7ffbZZ8vyF7/4xbI8MTERbXf69OmhHdNSSJdvT6fbzR49WvvanvCuuWqaUpe0jVy0JTiWX8Rt2y+KXzo5VZa7k8na8wAMXDEbT58vgvCu2Vvi5dtHn1kf1S/7369V2ybvu+Hjb/R3QLP10+uLkyej+ktfuLIsX3t/vMLni//puqi+9ftVWNiOR3/S37HBXPQsQx3/P7h7Jg6hDC1amEqDPEjNUCTh+ukzpqXfGZo05USSOqKpbwyl3zSELPX0m/A+tszCkgai4dqwaOFdTcLvwumxpGFpYcjgIp0bM34AAAAAWsrADwAAAEBLGfgBAAAAaKlW5/i57bbbyvIHPvCBsvzQQw8NdL9bt24tyzfccENZHh8fj7ZbsTl+muIMg7jXIs1/kMa1Bu8zcsO7o6bZn/y078Or00lyLBVT9bHrYVx5lmVZ9/jJmi0BGJTuiVO1bZ3N1bLnnRdfj9om3nVtVJ9960hZfvy1p6O2PVe9v79jOz1V2/ba5+KcQzuers8rcNOuOOfPO390VVXJ/X+OAUqe59I8WqHOunVRvSn/T9/SHBcNebR6+kYxvfjHA+eTfg9qWL59KfpNlO8q1XNPaXk+rIZzk4+OxZsOIv9Smg+q6Xg68bbFzOLnXPJEAQAAANBSBn4AAAAAWqp1oV67d+8uy/v27SvLr7zySll+8MEHB3oMH//4xwf6/gAAAABz0bqBH5ZWPlbFS+Zr4o/X7LHjUX3NpTvLcvfQm/3vtCnnUM2xZVmWFecaYjmTmMyBxH0C0Ld3Pnx9Wc6TsPktjz4X1YvHLynL1/+Pz0Vt12bPznmfxZmzc9pu6j1xHodtfz1au+1Fayej+uQTP6oqGyYyWFTB800+kuQGSZ51whwYxczMYI/rPMfTlA+jxxyfBaEv4feCJE9O2jfCftM9N/jcUz25YZpy/KTa0G/SPDqh5PcLv5sWM0PIC9aTi2we17QBEOoFAAAA0FKtm/Fz//33l+WJYBWnPXv2lOVTp+pXCenXtm3byvKtt95alrvdpR3ZAwAAAFav1g38MADzmAZYTFfTHfOmqXdZls1evr163f7D8z+uOWiaptw9fbr2dd3jJwZyPECLhde8NkyfXgrzmKK+9UfvlOWpKzY2bJllf/mePy/Le/bUL9/euUBoVdEwbf/cre8ty5f+eRzatfGJA2U5/XfQ/33sfVH9imx/sMN06yQUBhbgQiEh+Wj1NaE7Odmw5QJE1834894UXlY0LfUOiy26pzf/Uz/sNwNLFRH0m55+3K3vG63sN+G5ucB3zywMJx1C+GrPPbzh2XAY50aoFwAAAEBLrfgZP3fddVdU37t3b1l++eWXy/L+/fuzQfrSl75UlsPwrqeeeqosHzt2bKDHAAAAABAy4wcAAACgpVb8jB+WWCfON9BZt7Ysp/HgYb6dLMuy4rkD2ZxcKF6zSbjE4XR9boY1O3dE9ZnDb/W/T2B1ktdn4Rpi3DtXXxnVT123uSyPf+f5qO3cbTdG9b0fvDyovRG15SPB/8A6/d9vzm6r8vqMHU9zyk2V5TPfvixqu+KOhhnJae4GKX5YoHDJ9J4cOsnz1sDy+tRoyjnU8ww5qNwpcD7h9500h84S95umJcJXRb+ZR37F4uzZAR9MNufvreHS8lnWnNNssaz4gZ+77747qo+Pj5flr371qwPd965du8ryvffeW5ZngwfXr3zlK2V5umHgAQAAAGCxCfUCAAAAaKkVP+OHpdWZGI/qxdRUzZZZlt9wbbztDxtCvZqmyTVN40teF05p7p6pn943e+Ul8Q+aQr0WEnoGtEd6LVhOoV7L+diajNTHMr14/+ao/ksPnSjL6W/3xNf+a1T/6DW31L5vntzHItP1U687mzZE9cnt1f/SNv/4eNQWTsR/ave3orY9Wf3y8tmoxzQWpiecoCmccu3aqN49c2YgxxTJg/9BN4asJL9HG0NWWDbmE4az5P2mYfn2VvabJM1I4++fnJuhhHpFO2x49kqfd4R6nd/mzdXD3y231D/MPfLIIwM9js9+9rNlefv27WX5wIFqQOPJJ58c6DEAAAAA1BHqBQAAANBSBn4AAAAAWmpFhnqtDeL1Lr/88qht3759QzuOa6+99rw/f+GFF4Z2DEstWgY3y7JuGJ+YxGDmPzsc1Ueuv6Ysz770avLGnaCY5KpIYiKLc1W8amc8ydUQxlY2xICOvPl2VB98lOUyN4+lEWHVSvvGcuo3S73/fjXk+Hj3FXHuteKvj5Tl7vtvSLZ+Oqr9xat/VZb3XP1r8abbt5XF2c3ro6azF6+L6hPPHSrLr953TdS28WfV37z78mtZnfsO/eOo3tkQL7c79KWAabc8+R9vETzhpLnAOvG2a3buKMszC8l9GF6P0vwcddulTQ35tlal5XS/aaO03zRI8/+E30Uar+eL1W+a3qKF/Sb9Xtjw2JBlSU6zzsREWe6ePj2PnTbkTew392tDvrVBMeMHAAAAoKUM/AAAAAC01IoM9Tp58mRZfv7556O2G2+8sSxv21ZN33777TiUp1+XXFIt+33XXXedd5vvf//7i7IvAAAAgIVYkQM/LCPdJK44iEHNk1w82Zrk4/b2sbI4smlT8j5VvGQxmwRvTk9H1TzM65PES3bPnDnPQfeafetI8wb9xm+uVOLF4cJW23VhGNJ7SmC6G99Txt5T5dg5eMf6dPNaRz51c21bJ769ZONH4nvKoX9e7XPD6/Gxbv/WT8pymnIgH63ufy8/EOcjWnvqh/ExbNpQe3ywYA3XrXw87kfh81eePsM1KJJ+nI/ktW1N+Rej95yZvvBGq4nntMFqShyT9qEkH1CYe3Re/SbN+RLm9UmPZ47nf9X1m+TcpH//oiEXbaP079/0/DfXcyPHDwAAAACLZUXO+JmamirLr7zyStT2iU98oix/+9vfLssPP/zwvPaxe/fusnzNNdV/+Hbt2lWWi5oRvW63Kb04AAAAwHCsyIEflo908KszUYVd5ck0uNkjcThVZ121TO5cQ7IWU752bVkuzp4d+v6BFc5U+0VX9w+VLMuyk2fXRvXJ362maa//Xny/2XPV+6P6zU9X95iL/9uzfR/f+sercr4uPp7umfr7yImPVfkHN37zub73D/OWhCj0hOEHZo/+It42CJNIl6xewCHMnSXLWSaivpCEKxbT55KNF+lzW/QZCrTa+k1DyFbP98ul/tss8f6FegEAAAC01Iqf8fPAAw9E9XCWyR133FGW9+3bN6/3PXr0aFkO/wO5ffv2C772a1/72rz2BQAAADAIZvwAAAAAtNSKn/HDEDQsWVcEibazLI4BH7n44vht0iUNgzjzTrgk+4X2nyx/1xR33tQ2svOSstwNlpbPsizrnjxZfzwADER6n+hsnyjLW74S3ye2PfdiWX7r0aujtjMvvS+qP7jjv1SV1+J9fuynHy3LX7jiiXj/ycLsD/zOvyrLGw/G97/OMweyOh+9/6my/MfX/9Oo7V0P7a99HSxUz5LBwSz2MNdhlmU9z1ehhSzn3pjkpyHPRT42Vm02nTzPzXEZeOhHY78ZHYub0s9iuLx7w6rfvTvtr5+k8jWj1cvS36MF/abpu13P8u3pnzQ8N9kCFmPqMz9PmGOt6TM2KCt+4OfFF1+M6vfcc09Zvummm8ryddddN6/3/cY3vnHen3/9618vy/fee+95t5lKBkMAAAAAloJQLwAAAICWWvEzflhana1ba9uKZAm9niltwRLq6XTjcHpvzzy9y3fGx3C2WkZx5tDfNB1ufDzj1XLy2ZGVP/VxWWgIC1wVS0pCP/SbSif+WxTT02X5pX8Z3yc2/Na1ZfnUGxuitst+FN8Lbr/7U2X5jQ/GIWNT/6C6V+15d/2S7FmWZbf/2z8sy3v/2T1RW9Ok8fu3V7OTH3v7tsZ9wGLqpOFcgZ6QieR6Ez639SwDH4ZMJM9pnYkkfD94n+7kZMPRJrsIro2r7Eo4OO43cxJ9D0k1hERmWRb3hzyeY5F36v/+nY1b4rc5V32/6Z4+3bzPcB8j1T57vnu1QFPYaU+YaSoMdUv7Ql4/H2ZkU/yMEYaezufcxPsY/rlp9cDP888/f97yQrz66qsX3Gb37t1l+YUXXliU/QIAAADMl1AvAAAAgJYy8AMAAADQUq0O9RqEMN44r4mTbX14VxADPHv0aNTUWb++LOdjo1HbyJY4djWM8y7OJnkVOsGYZDeOHc/fOZEcTtA+j2UKi9cPV+V0mdCmGGjqiQ+H+dNvahWT1SqZN/x+nLfn7X9yRVnu7Iz/j/WT3740ql/6f6r8JO/6i2NR29G3NpflD/3JfVHbkV+P85GMranuMTtfPdh06JH3/vvPl+WLDk5HbZ0kdwAspm6SbzHrVH0hzduT5lsMn43SXCF5kvInet2Z5lxZc9U9F/SVpqWumTv3mznp/V4S9JskT0/P8u4zwec2+V5SZPUdZz75r5q0vd/05CYLv7OleXo6yd87/Hv09IX6v1V36kxt23xEn40l6Itm/AAAAAC0lIEfAAAAgJYS6jVPRTAtq1it0yUbwqCiaYpTyVTIdCnQoJ4uv5cH4V3p69LwsnCqXGdiIj6ehiX28vEqLC1dmjGaigfAksjXryvLRTINfss3q9U6twZhxlmWZZdec1lUn7yyujfk5+Jp4nkwu/v41XGI8lWffTOqd0+cKstvfu5Xo7adj+zvOf6/t+6D1X3rnbPbo7bxZ5J76mp9tmA4gtCTIg2Pb1reuOlzmYRXFNPnajbsXYq5J2wj3DZ8Tkwj+Xt+AAMU9ZsLbDvHdBFpyFhPeFkoDVlqSG0Rvm/RTeZ4tLHfhNemC/1+c03lkV7Tms5N+p4N18r4mpYc6xDu/Wb8AAAAALSUgR8AAACAljLwAwAAANBScvzM07p1687786mpqfP+fNVpiJ3siWVM64HGJQ3TGMhgn52N8bK40fukr9uyqdru50fmvA8All4+MV7b1jkY5+bZcLD+fbb/2dHatjCnT5bFS69fd/dPo7apR7eW5dlfvBO1PXPzo2X5js99LN7HZPz8EOY1gqFKn30a8mU05eZpMr8cP9X/p5vyBsGy0pSrZT65ggJpvtOe/FyhcNuZhtw0q1HTuZlPrqBAz7lpujZGedP6u4YuhBk/AAAAAC1l4AcAAACgpYR6zdOnP/3psnzs2LGy/OUvf3kpDgcAAACgloEfVr4gJrN77HjUFMZdpjGXxetBDgg5fQC4gO6p02X56IM3Rm2Te6pHqm1/9GzUduO/+3xZvuzw/qgtzBsEbdc9Nx3/IHzeSp7FeraFVaqYmXu/KaaHnztmNevJYdu0bXoeh0yoFwAAAEBLmfEzT88880xZfvjhh8vyk08+uRSHAwAAAFDLwA8r3ppLd5bl4ly83GferaY/zr4TL6/bPX06Y8AapqICNfSb5aMTh/0+/v9/ENR+kNXZ+61bo/pl/2F/zZbQfp3x8arSjdewLoJrXHE2WXq6aclqFkdTagP3nyXV2bixqkzHIULFbNWPiun4u49+M3idiYmq0hBq13NulrhPCfUCAAAAaCkzfubpzjvvXOpDAAAAAJgTM34AAAAAWsqMH1a87omTZTkfjT/SeRiD+UtXxi/8qx8N8rDIsuZYVnlM4Pzm2m/m8zr6043/pr/8H6tl2e/7F9+J2v7N1oNl+eyvXBe1jX7vx4t/bLBCRPkX8/h/zvnYaFUeGYnaupOTAz0uMveNZax7uvr850m+uXxsrKokbT25slh03akztW3hd9F8Tfy9tJiZSTcfKjN+AAAAAFrKwA8AAABASwn1YsXJR8eiemf7trJ87sqLorbu2mra8NoXDkVtFjscgjAsJW8YZy6cDSg19JtwuvdSTxleDc7d+t6oPnPTqbIchnalumuSkLxwKn5XaAUtl4Sk5uvXl+XO5k21284e/vlAD4sL6IzUt1kifPCSftNZv65q2jARb9qpng1mjxwd7HHRe00Lw7nG4u+l+Uhwbk6cypYTM34AAAAAWsrADwAAAEBLGfgBAAAAaKm8sIwfAAAAQCuZ8QMAAADQUgZ+AAAAAFrKwA8AAABASxn4AQAAAGgpAz8AAAAALWXgBwAAAKClDPwAAAAAtJSBHwAAAICWMvADAAAA0FIGfgAAAABaysAPAAAAQEsZ+AEAAABoKQM/AAAAAC1l4AcAAACgpQz8AAAAALSUgR8AAACAljLwAwAAANBSBn4AAAAAWsrADwAAAEBLGfgBAAAAaCkDPwAAAAAtZeAHAAAAoKUM/AAAAAC01N8CoXmhNFG2bzAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1440x720 with 55 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "batch_size=1\n",
    "n_iterations_test = len(mnist.test.labels) // batch_size\n",
    "checkpoint_path = \"./tensorboard/Models/\"+folder+\"/my_capsule_network\"\n",
    "\n",
    "caps_and_routing = [convcaps_output, routing1, caps1_output]\n",
    "caps_and_routing_dims = [24, 24, 32, 10]\n",
    "reduce = [2]\n",
    "\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, checkpoint_path)\n",
    "    i = 1\n",
    "    columns, rows = 11, 5\n",
    "    fig=plt.figure(figsize=(20, 10))\n",
    "    for iteration in range(1, n_iterations_test + 1)[:rows]:\n",
    "        X_batch = mnist.test.images[(batch_size*(iteration-1)):(batch_size*iteration)]\n",
    "        y_batch = mnist.test.labels[(batch_size*(iteration-1)):(batch_size*iteration)]\n",
    "        [rpv_raw] = sess.run([visualize_convcaps(caps_and_routing, caps_and_routing_dims, reduce)], \n",
    "                             feed_dict={X: X_batch.reshape([-1, 28, 28, 1]),\n",
    "                                        y: y_batch}) \n",
    " \n",
    "\n",
    "        fig.add_subplot(rows, columns, i)\n",
    "        plt.imshow(X_batch[0].reshape(28,28), cmap='gray')\n",
    "        plt.axis(\"off\")\n",
    "\n",
    "\n",
    "        for n in range(1, 11):\n",
    "            rpv = rpv_raw[:,:,n-1]\n",
    "            fig.add_subplot(rows, columns, i+n)\n",
    "            plt.imshow(rpv, vmin=0, vmax=0.000015)\n",
    "            plt.axis(\"off\")\n",
    "        i = i+11\n",
    "    plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}