multi-armed-bandit.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# The Multi Armed Bandit Problem\n",
    "\n",
    "### Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### n-armed bandit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "bandits = [0.2, 0, -0.2, -5]\n",
    "num_bandits = len(bandits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def pull_bandit(bandit):\n",
    "    \"\"\"\n",
    "    Pull a bandit.\n",
    "    \n",
    "    Generates a random number from a normal\n",
    "    distribution with mean of 0.\n",
    "    \n",
    "    :param bandit: float\n",
    "        The bandit to be pulled.\n",
    "    \n",
    "    :return reward: int\n",
    "        The reward for pulling a bandit.\n",
    "        The lower the bandit the more likely it\n",
    "        is for a positive reward.\n",
    "    \"\"\"\n",
    "    reward = 1 if np.random.randn(1) > bandit else -1\n",
    "    return reward"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## The Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "# Weights: contains the value that estimates how good a \n",
    "#          choosen (pulled) bandit is.\n",
    "# Action:  is the action that gives the maximum \n",
    "#          weight value\n",
    "weights = tf.Variable(tf.ones(shape=[num_bandits]))\n",
    "choosen_action = tf.argmax(weights, axis=0)\n",
    "\n",
    "# Reward: reward from pulling a bandit/lever\n",
    "# Action: the action that lead to the reward\n",
    "reward_placeholder = tf.placeholder(tf.float32, shape=[1])\n",
    "action_placeholder = tf.placeholder(tf.int32, shape=[1])\n",
    "# action slice from the weight\n",
    "responsible_weight = tf.slice(weights, action_placeholder, [1])\n",
    "\n",
    "# loss = -log(π) * A\n",
    "loss = -tf.log(responsible_weight) * reward_placeholder\n",
    "\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate=1e-3)\n",
    "train = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training the Agent"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tensorflow's `Session`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess = tf.Session()\n",
    "init = tf.global_variables_initializer()\n",
    "sess.run(init)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "episodes = 1000\n",
    "log_interval = 100\n",
    "rewards = np.zeros(shape=num_bandits)\n",
    "e = 0.7  # Epsilon:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "* - random action\n",
      "0: Bandit 0 looks good\tRewards: [-1.  0.  0.  0.]\n",
      "100: Bandit 0 looks good\tRewards: [ -3.   1.  -2.  75.]*\n",
      "200: Bandit 3 looks good\tRewards: [  -4.    5.    1.  155.]\n",
      "300: Bandit 3 looks good\tRewards: [  -3.    4.    5.  239.]\n",
      "400: Bandit 3 looks good\tRewards: [  -1.    6.    5.  323.]\n",
      "500: Bandit 3 looks good\tRewards: [  -4.    9.    2.  404.]\n",
      "600: Bandit 3 looks good\tRewards: [  -5.    8.    7.  487.]\n",
      "700: Bandit 3 looks good\tRewards: [  -6.   11.    9.  565.]\n",
      "800: Bandit 0 looks good\tRewards: [ -11.   13.   16.  647.]*\n",
      "900: Bandit 3 looks good\tRewards: [ -14.   13.   13.  731.]\n",
      "\n",
      "Agent says bandit 3 is the best...\n",
      "...and it was right!\n"
     ]
    }
   ],
   "source": [
    "print('* - random action')\n",
    "\n",
    "for i in range(episodes):\n",
    "    # Explore or Exploit\n",
    "    if np.random.randn(1) > e:\n",
    "        action = np.random.randint(num_bandits)\n",
    "        is_random = True\n",
    "    else:\n",
    "        action = sess.run(choosen_action)\n",
    "        is_random = False\n",
    "    # Reward from taking that action\n",
    "    reward = pull_bandit(bandits[action])\n",
    "    # Train the net\n",
    "    _, _weight = sess.run([train, weights], \n",
    "                          feed_dict={reward_placeholder:[reward],\n",
    "                                     action_placeholder:[action]})\n",
    "    rewards[action] += reward\n",
    "    # Logging\n",
    "    if i % log_interval == 0:\n",
    "        rand_msg = '*' if is_random else ''\n",
    "        print(f'{i}: Bandit {action} looks good\\tRewards: {rewards}{rand_msg}')\n",
    "    if i+1 == episodes:\n",
    "        print(f'\\nAgent says bandit {action} is the best...')\n",
    "        if np.argmax(_weight) == np.argmax(-np.array(bandits)):\n",
    "            print(f'...and it was right!')\n",
    "        else:\n",
    "            print(f'...and it was wrong!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}