ea_naive.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" To run this notebook clone the repository found at https://github.com/OhGreat/evolutionary_algorithms\n",
    "    and modify the imports relative to where you cloned the repository.\n",
    "\"\"\"\n",
    "\n",
    "import sys\n",
    "sys.path.append('evolutionary_algorithms/')\n",
    "\n",
    "import gym\n",
    "import numpy as np\n",
    "from evolutionary_algorithms.classes.Population import *\n",
    "from evolutionary_algorithms.classes.EA import *\n",
    "from evolutionary_algorithms.classes.Recombination import *\n",
    "from evolutionary_algorithms.classes.Mutation import *\n",
    "from evolutionary_algorithms.classes.Selection import *\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h4>Reward Maximization evaluation function</h4>\n",
    "This class is used to evaluate the individuals and get their fitness."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RewardMaximization():\n",
    "    def __init__(self, env, reps=3):\n",
    "        self.env = env\n",
    "        self.n_observations = np.sum([dim for dim in env.observation_space.shape]) \n",
    "        self.n_actions = env.action_space.n\n",
    "        self.reps = reps\n",
    "\n",
    "    def __call__(self, population):\n",
    "        fitnesses = []\n",
    "        for individual in population.individuals:\n",
    "            ind_rews = []\n",
    "            for i in range(self.reps):\n",
    "                state = self.env.reset()\n",
    "                rep_rews = 0\n",
    "                done = False\n",
    "                while not done:\n",
    "                    # sample action\n",
    "                    a = np.argmax(np.dot(individual.reshape(self.n_actions, \n",
    "                                                        self.n_observations), \n",
    "                                                        state))                    \n",
    "                    # query environment\n",
    "                    state, rew, done, _ = self.env.step(a)\n",
    "                    rep_rews += rew\n",
    "                ind_rews.append(rep_rews)\n",
    "            fitnesses.append(np.mean(ind_rews))\n",
    "        population.fitnesses = np.array(fitnesses)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h4>Creating gym environment</h4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define population size \n",
    "env = gym.make('CartPole-v1')\n",
    "n_observations = np.sum([dim for dim in env.observation_space.shape]) \n",
    "n_actions = env.action_space.n\n",
    "\n",
    "pop_size = 4\n",
    "ind_size = n_observations * n_actions "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h4>Define Evolutionary Algorithm parameters</h4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "minimize = False\n",
    "budget = 200\n",
    "patience = 5\n",
    "parents_size = 2\n",
    "offspring_size = 4\n",
    "individual_size = ind_size\n",
    "recomb = Discrete()\n",
    "mutation = IndividualSigma()\n",
    "selection = PlusSelection()\n",
    "evaluation = RewardMaximization(env, reps=10)\n",
    "repetitions = 10\n",
    "v = 1\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h4>Run the EA for a predefined number of repetitions and collect results</h4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Curr iteration: 0\n",
      "Best eval: 500.0 on budget: 14\n",
      "Curr iteration: 1\n",
      "Best eval: 500.0 on budget: 2\n",
      "Curr iteration: 2\n",
      "Best eval: 500.0 on budget: 26\n",
      "Curr iteration: 3\n",
      "Best eval: 500.0 on budget: 14\n",
      "Curr iteration: 4\n",
      "Best eval: 10.2 on budget: 58\n",
      "Curr iteration: 5\n",
      "Best eval: 495.6 on budget: 178\n",
      "Curr iteration: 6\n",
      "Best eval: 500.0 on budget: 42\n",
      "Curr iteration: 7\n",
      "Best eval: 500.0 on budget: 82\n",
      "Curr iteration: 8\n",
      "Best eval: 500.0 on budget: 62\n",
      "Curr iteration: 9\n",
      "Best eval: 500.0 on budget: 102\n"
     ]
    }
   ],
   "source": [
    "best_results = []\n",
    "for i in range(repetitions):\n",
    "    print(f\"Curr iteration: {i}\")\n",
    "    ea = EA(minimize=minimize, budget=budget, patience=patience, \n",
    "        parents_size=parents_size, offspring_size=offspring_size,\n",
    "        individual_size=individual_size, recombination=recomb,\n",
    "        mutation=mutation, selection=selection, evaluation=evaluation,\n",
    "        verbose=v)\n",
    "    best_ind, best_eval, _ = ea.run()\n",
    "    best_results.append([best_ind, best_eval])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h4>Evaluate the trained agent.</h4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def eval(individual, env, render=False):\n",
    "    \"\"\" Simulates a playthrough of our agent with the environment \n",
    "    \"\"\"\n",
    "    n_observations = np.sum([dim for dim in env.observation_space.shape]) \n",
    "    n_actions = env.action_space.n\n",
    "\n",
    "    done = False\n",
    "    state = env.reset()\n",
    "    if render:\n",
    "        env.render()\n",
    "        \n",
    "    tot_rew = 0\n",
    "    done = False\n",
    "    while not done:\n",
    "        # Sample action\n",
    "        a = np.argmax(np.dot(individual.reshape(n_actions, \n",
    "                                                n_observations), \n",
    "                                                state))\n",
    "        # query environment\n",
    "        state, rew, done, _ = env.step(a)\n",
    "        tot_rew += rew\n",
    "        if render:\n",
    "            env.render()\n",
    "    return tot_rew"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average results of 100 runs for each agent: [500.0, 500.0, 500.0, 500.0, 500.0, 500.0, 500.0, 500.0, 500.0, 500.0]\n"
     ]
    }
   ],
   "source": [
    "# Main evaluation loop\n",
    "final_evals = []\n",
    "for ind in best_results:\n",
    "    evals = []\n",
    "    for i in range(100):\n",
    "        evals.append(eval(best_ind, env, render=False))\n",
    "    final_evals.append(np.mean(evals))\n",
    "\n",
    "print(\"Average results of 100 runs for each agent:\",final_evals)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "chosen individual: 0\n",
      "[0.50603473 0.3420444  0.05768099 0.55968364 0.56565455 0.45848474\n",
      " 0.51473977 0.91867617]\n"
     ]
    }
   ],
   "source": [
    "# Save best individual to file\n",
    "best_ind_idx = np.argmax(final_evals)\n",
    "print(f\"chosen individual: {best_ind_idx}\")\n",
    "best_ind = best_results[best_ind_idx][0]\n",
    "np.save('es_naive.npy', best_ind)\n",
    "print(best_ind)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h4>Do a final evaluation on the best chosen individual</h4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "500.0\n"
     ]
    }
   ],
   "source": [
    "# load trained weights\n",
    "a = np.load('es_naive.npy')\n",
    "\n",
    "# simulate 100 runs to average results for the individual\n",
    "final_evals = []\n",
    "for i in range(100):\n",
    "    evals.append(eval(a, env, render=False))\n",
    "    final_evals.append(np.mean(evals))\n",
    "print(np.mean(final_evals))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "3e3d537f7c047eb69d35d93db02503d7f99d5e4e21342e5ae19b3dd47808d68e"
  },
  "kernelspec": {
   "display_name": "Python 3.9.12 ('pytorch')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}