marketEnvForRLprojects.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 47,
      "id": "w0UUBPWSt0uu",
      "metadata": {
        "id": "w0UUBPWSt0uu"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 48,
      "id": "013089f3",
      "metadata": {},
      "outputs": [],
      "source": [
        "from collections import namedtuple, deque\n",
        "import math\n",
        "import random\n",
        "import wandb"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "b200e6e9",
      "metadata": {},
      "source": [
        "## ENV"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 49,
      "id": "2Ee6dqCMp3jE",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2Ee6dqCMp3jE",
        "outputId": "3d5836cb-97dc-4ba3-cb55-6f35e25abc25"
      },
      "outputs": [],
      "source": [
        "!pip install -q --upgrade --force-reinstall --no-deps kaggle"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 50,
      "id": "mAVLcFUWp5Qy",
      "metadata": {
        "id": "mAVLcFUWp5Qy"
      },
      "outputs": [],
      "source": [
        "# from google.colab import files"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 51,
      "id": "3gwQ2ovBp-Dj",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 91,
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7CgpmdW5jdGlvbiBfdXBsb2FkRmlsZXMoaW5wdXRJZCwgb3V0cHV0SWQpIHsKICBjb25zdCBzdGVwcyA9IHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCk7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICAvLyBDYWNoZSBzdGVwcyBvbiB0aGUgb3V0cHV0RWxlbWVudCB0byBtYWtlIGl0IGF2YWlsYWJsZSBmb3IgdGhlIG5leHQgY2FsbAogIC8vIHRvIHVwbG9hZEZpbGVzQ29udGludWUgZnJvbSBQeXRob24uCiAgb3V0cHV0RWxlbWVudC5zdGVwcyA9IHN0ZXBzOwoKICByZXR1cm4gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpOwp9CgovLyBUaGlzIGlzIHJvdWdobHkgYW4gYXN5bmMgZ2VuZXJhdG9yIChub3Qgc3VwcG9ydGVkIGluIHRoZSBicm93c2VyIHlldCksCi8vIHdoZXJlIHRoZXJlIGFyZSBtdWx0aXBsZSBhc3luY2hyb25vdXMgc3RlcHMgYW5kIHRoZSBQeXRob24gc2lkZSBpcyBnb2luZwovLyB0byBwb2xsIGZvciBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcC4KLy8gVGhpcyB1c2VzIGEgUHJvbWlzZSB0byBibG9jayB0aGUgcHl0aG9uIHNpZGUgb24gY29tcGxldGlvbiBvZiBlYWNoIHN0ZXAsCi8vIHRoZW4gcGFzc2VzIHRoZSByZXN1bHQgb2YgdGhlIHByZXZpb3VzIHN0ZXAgYXMgdGhlIGlucHV0IHRvIHRoZSBuZXh0IHN0ZXAuCmZ1bmN0aW9uIF91cGxvYWRGaWxlc0NvbnRpbnVlKG91dHB1dElkKSB7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICBjb25zdCBzdGVwcyA9IG91dHB1dEVsZW1lbnQuc3RlcHM7CgogIGNvbnN0IG5leHQgPSBzdGVwcy5uZXh0KG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSk7CiAgcmV0dXJuIFByb21pc2UucmVzb2x2ZShuZXh0LnZhbHVlLnByb21pc2UpLnRoZW4oKHZhbHVlKSA9PiB7CiAgICAvLyBDYWNoZSB0aGUgbGFzdCBwcm9taXNlIHZhbHVlIHRvIG1ha2UgaXQgYXZhaWxhYmxlIHRvIHRoZSBuZXh0CiAgICAvLyBzdGVwIG9mIHRoZSBnZW5lcmF0b3IuCiAgICBvdXRwdXRFbGVtZW50Lmxhc3RQcm9taXNlVmFsdWUgPSB2YWx1ZTsKICAgIHJldHVybiBuZXh0LnZhbHVlLnJlc3BvbnNlOwogIH0pOwp9CgovKioKICogR2VuZXJhdG9yIGZ1bmN0aW9uIHdoaWNoIGlzIGNhbGxlZCBiZXR3ZWVuIGVhY2ggYXN5bmMgc3RlcCBvZiB0aGUgdXBsb2FkCiAqIHByb2Nlc3MuCiAqIEBwYXJhbSB7c3RyaW5nfSBpbnB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIGlucHV0IGZpbGUgcGlja2VyIGVsZW1lbnQuCiAqIEBwYXJhbSB7c3RyaW5nfSBvdXRwdXRJZCBFbGVtZW50IElEIG9mIHRoZSBvdXRwdXQgZGlzcGxheS4KICogQHJldHVybiB7IUl0ZXJhYmxlPCFPYmplY3Q+fSBJdGVyYWJsZSBvZiBuZXh0IHN0ZXBzLgogKi8KZnVuY3Rpb24qIHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IGlucHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKGlucHV0SWQpOwogIGlucHV0RWxlbWVudC5kaXNhYmxlZCA9IGZhbHNlOwoKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIG91dHB1dEVsZW1lbnQuaW5uZXJIVE1MID0gJyc7CgogIGNvbnN0IHBpY2tlZFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgaW5wdXRFbGVtZW50LmFkZEV2ZW50TGlzdGVuZXIoJ2NoYW5nZScsIChlKSA9PiB7CiAgICAgIHJlc29sdmUoZS50YXJnZXQuZmlsZXMpOwogICAgfSk7CiAgfSk7CgogIGNvbnN0IGNhbmNlbCA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2J1dHRvbicpOwogIGlucHV0RWxlbWVudC5wYXJlbnRFbGVtZW50LmFwcGVuZENoaWxkKGNhbmNlbCk7CiAgY2FuY2VsLnRleHRDb250ZW50ID0gJ0NhbmNlbCB1cGxvYWQnOwogIGNvbnN0IGNhbmNlbFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgY2FuY2VsLm9uY2xpY2sgPSAoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9OwogIH0pOwoKICAvLyBXYWl0IGZvciB0aGUgdXNlciB0byBwaWNrIHRoZSBmaWxlcy4KICBjb25zdCBmaWxlcyA9IHlpZWxkIHsKICAgIHByb21pc2U6IFByb21pc2UucmFjZShbcGlja2VkUHJvbWlzZSwgY2FuY2VsUHJvbWlzZV0pLAogICAgcmVzcG9uc2U6IHsKICAgICAgYWN0aW9uOiAnc3RhcnRpbmcnLAogICAgfQogIH07CgogIGNhbmNlbC5yZW1vdmUoKTsKCiAgLy8gRGlzYWJsZSB0aGUgaW5wdXQgZWxlbWVudCBzaW5jZSBmdXJ0aGVyIHBpY2tzIGFyZSBub3QgYWxsb3dlZC4KICBpbnB1dEVsZW1lbnQuZGlzYWJsZWQgPSB0cnVlOwoKICBpZiAoIWZpbGVzKSB7CiAgICByZXR1cm4gewogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgICAgfQogICAgfTsKICB9CgogIGZvciAoY29uc3QgZmlsZSBvZiBmaWxlcykgewogICAgY29uc3QgbGkgPSBkb2N1bWVudC5jcmVhdGVFbGVtZW50KCdsaScpOwogICAgbGkuYXBwZW5kKHNwYW4oZmlsZS5uYW1lLCB7Zm9udFdlaWdodDogJ2JvbGQnfSkpOwogICAgbGkuYXBwZW5kKHNwYW4oCiAgICAgICAgYCgke2ZpbGUudHlwZSB8fCAnbi9hJ30pIC0gJHtmaWxlLnNpemV9IGJ5dGVzLCBgICsKICAgICAgICBgbGFzdCBtb2RpZmllZDogJHsKICAgICAgICAgICAgZmlsZS5sYXN0TW9kaWZpZWREYXRlID8gZmlsZS5sYXN0TW9kaWZpZWREYXRlLnRvTG9jYWxlRGF0ZVN0cmluZygpIDoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ24vYSd9IC0gYCkpOwogICAgY29uc3QgcGVyY2VudCA9IHNwYW4oJzAlIGRvbmUnKTsKICAgIGxpLmFwcGVuZENoaWxkKHBlcmNlbnQpOwoKICAgIG91dHB1dEVsZW1lbnQuYXBwZW5kQ2hpbGQobGkpOwoKICAgIGNvbnN0IGZpbGVEYXRhUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICAgIGNvbnN0IHJlYWRlciA9IG5ldyBGaWxlUmVhZGVyKCk7CiAgICAgIHJlYWRlci5vbmxvYWQgPSAoZSkgPT4gewogICAgICAgIHJlc29sdmUoZS50YXJnZXQucmVzdWx0KTsKICAgICAgfTsKICAgICAgcmVhZGVyLnJlYWRBc0FycmF5QnVmZmVyKGZpbGUpOwogICAgfSk7CiAgICAvLyBXYWl0IGZvciB0aGUgZGF0YSB0byBiZSByZWFkeS4KICAgIGxldCBmaWxlRGF0YSA9IHlpZWxkIHsKICAgICAgcHJvbWlzZTogZmlsZURhdGFQcm9taXNlLAogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbnRpbnVlJywKICAgICAgfQogICAgfTsKCiAgICAvLyBVc2UgYSBjaHVua2VkIHNlbmRpbmcgdG8gYXZvaWQgbWVzc2FnZSBzaXplIGxpbWl0cy4gU2VlIGIvNjIxMTU2NjAuCiAgICBsZXQgcG9zaXRpb24gPSAwOwogICAgZG8gewogICAgICBjb25zdCBsZW5ndGggPSBNYXRoLm1pbihmaWxlRGF0YS5ieXRlTGVuZ3RoIC0gcG9zaXRpb24sIE1BWF9QQVlMT0FEX1NJWkUpOwogICAgICBjb25zdCBjaHVuayA9IG5ldyBVaW50OEFycmF5KGZpbGVEYXRhLCBwb3NpdGlvbiwgbGVuZ3RoKTsKICAgICAgcG9zaXRpb24gKz0gbGVuZ3RoOwoKICAgICAgY29uc3QgYmFzZTY0ID0gYnRvYShTdHJpbmcuZnJvbUNoYXJDb2RlLmFwcGx5KG51bGwsIGNodW5rKSk7CiAgICAgIHlpZWxkIHsKICAgICAgICByZXNwb25zZTogewogICAgICAgICAgYWN0aW9uOiAnYXBwZW5kJywKICAgICAgICAgIGZpbGU6IGZpbGUubmFtZSwKICAgICAgICAgIGRhdGE6IGJhc2U2NCwKICAgICAgICB9LAogICAgICB9OwoKICAgICAgbGV0IHBlcmNlbnREb25lID0gZmlsZURhdGEuYnl0ZUxlbmd0aCA9PT0gMCA/CiAgICAgICAgICAxMDAgOgogICAgICAgICAgTWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCk7CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPSBgJHtwZXJjZW50RG9uZX0lIGRvbmVgOwoKICAgIH0gd2hpbGUgKHBvc2l0aW9uIDwgZmlsZURhdGEuYnl0ZUxlbmd0aCk7CiAgfQoKICAvLyBBbGwgZG9uZS4KICB5aWVsZCB7CiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICB9CiAgfTsKfQoKc2NvcGUuZ29vZ2xlID0gc2NvcGUuZ29vZ2xlIHx8IHt9OwpzY29wZS5nb29nbGUuY29sYWIgPSBzY29wZS5nb29nbGUuY29sYWIgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYi5fZmlsZXMgPSB7CiAgX3VwbG9hZEZpbGVzLAogIF91cGxvYWRGaWxlc0NvbnRpbnVlLAp9Owp9KShzZWxmKTsK",
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "ok": true,
              "status": 200,
              "status_text": ""
            }
          }
        },
        "id": "3gwQ2ovBp-Dj",
        "outputId": "e4732116-b6c4-4ea8-d28c-feb114ef2687"
      },
      "outputs": [],
      "source": [
        "# files.upload()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 52,
      "id": "3UkUzcxDp_2Z",
      "metadata": {
        "id": "3UkUzcxDp_2Z"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "mkdir: cannot create directory ‘/home/anirudh/.kaggle’: File exists\n"
          ]
        }
      ],
      "source": [
        "!mkdir ~/.kaggle"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 53,
      "id": "bTrnSbLpqBwe",
      "metadata": {
        "id": "bTrnSbLpqBwe"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "mv: cannot stat './kaggle.json': No such file or directory\n"
          ]
        }
      ],
      "source": [
        "!mv ./kaggle.json ~/.kaggle/."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 54,
      "id": "ZAs2ql1cqDkR",
      "metadata": {
        "id": "ZAs2ql1cqDkR"
      },
      "outputs": [],
      "source": [
        "!chmod 600 ~/.kaggle/kaggle.json"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 57,
      "id": "0kdYNw4IqHSo",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0kdYNw4IqHSo",
        "outputId": "032fe269-4e1c-458c-903a-17d5d2fa4bc3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "ref                                title                     size  lastUpdated          downloadCount  voteCount  usabilityRating  \n",
            "---------------------------------  -----------------------  -----  -------------------  -------------  ---------  ---------------  \n",
            "datasets/gmshroff/market-data-cl   Market Data CL           404MB  2022-02-11 12:24:49             13          1  0.23529412       \n",
            "datasets/gmshroff/market-data      Market Data               39MB  2022-02-07 11:11:00             59          1  0.125            \n",
            "datasets/gmshroff/few-shot-arc     Few-shot version of ARC  609KB  2022-02-14 11:03:57             82          0  0.25             \n",
            "datasets/gmshroff/options-dataset  Options Dataset          151MB  2022-02-07 17:01:18              5          0  0.0              \n",
            "datasets/gmshroff/market-data-rl   Market Data RL           148MB  2022-02-09 12:25:02             47          1  0.29411766       \n",
            "datasets/gmshroff/few-shot-nar     Few-shot version of NAR  370MB  2022-02-01 09:29:05             59          0  0.3125           \n"
          ]
        }
      ],
      "source": [
        "!kaggle datasets list --user gmshroff"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "U34YsqUxqHs9",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U34YsqUxqHs9",
        "outputId": "5669f07c-e764-4f5d-d9e8-0aee9748f7a1"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Downloading market-data-rl.zip to /home/anirudh/test\n",
            "100%|███████████████████████████████████████▉| 148M/148M [00:45<00:00, 5.05MB/s]\n",
            "100%|████████████████████████████████████████| 148M/148M [00:45<00:00, 3.44MB/s]\n"
          ]
        }
      ],
      "source": [
        "!kaggle datasets download -d gmshroff/market-data-rl"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "0GfOhup3qnhL",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0GfOhup3qnhL",
        "outputId": "e98f5990-53c1-434e-a203-b81d0826fe6b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Archive:  market-data-rl.zip\n",
            "  inflating: dfrl.csv                \n"
          ]
        }
      ],
      "source": [
        "!unzip market-data-rl"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 56,
      "id": "fsF8sK6-quBp",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fsF8sK6-quBp",
        "outputId": "3f33159a-e7b0-497e-ed11-8036a9d25891"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "__pycache__  fqn.pt\t\t\t   meta-envs  train.py\n",
            "close.png    log.txt\t\t\t   models     train_contrast.py\n",
            "dfrl.csv     market-data-rl.zip\t\t   og.png     visualise.py\n",
            "far.png      marketEnvForRLprojects.ipynb  psi.pt     wandb\n"
          ]
        }
      ],
      "source": [
        "!ls"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 58,
      "id": "0e656fa2-428a-47ce-91de-6430f202c466",
      "metadata": {
        "id": "0e656fa2-428a-47ce-91de-6430f202c466"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 59,
      "id": "09d59684-93c1-4ba3-9ff5-f7af67b659ee",
      "metadata": {
        "id": "09d59684-93c1-4ba3-9ff5-f7af67b659ee"
      },
      "outputs": [],
      "source": [
        "class marketEnv():\n",
        "    def __init__(self,df,symbol,day,target,stoploss,txcost=.001):\n",
        "        self.df=df.loc[(df['day']==day)&(df['sym']==symbol)]\n",
        "        self.symbol=symbol\n",
        "        self.day=day\n",
        "        self.t=target\n",
        "        self.l=stoploss\n",
        "        self.sl=stoploss\n",
        "        self.time=0\n",
        "        self.end=self.df.shape[0]\n",
        "        self.r=self.df['Close'].values\n",
        "        self.done=False\n",
        "        self.mv=self.df.iloc[0]['Open']\n",
        "        self.txcost=txcost\n",
        "    def thresh(self,x,pos):\n",
        "        #x=current_price,pos=position_taken(+-)\n",
        "        if pos>0:\n",
        "            if x>pos+self.t*pos or x<pos-self.l*pos: return 1\n",
        "            else: return 0\n",
        "        elif pos<0:\n",
        "            if x<abs(pos)-self.t*abs(pos) or x>abs(pos)+self.l*abs(pos): return 1\n",
        "            else: return 0\n",
        "    def partial_thresh(self,x,pos):\n",
        "        #x=current_price,pos=position_taken(+-)\n",
        "        if pos>0:\n",
        "            if x>pos+self.t*pos/2: return 1\n",
        "            else: return 0\n",
        "        elif pos<0:\n",
        "            if x<abs(pos)-self.t*abs(pos)/2: return 1\n",
        "            else: return 0\n",
        "    def get_state(self):\n",
        "        return self.df.iloc[0:self.time+1]\n",
        "    def step(self,action):\n",
        "        if action==0: \n",
        "            ret=0.0\n",
        "            if self.time+1==self.end: self.done=True\n",
        "            else: self.time+=1\n",
        "            return self.df.iloc[0:self.time+1],ret,self.done\n",
        "        else: \n",
        "            r=self.r\n",
        "            pos=action*r[self.time]\n",
        "            while True:\n",
        "                thresh_met=self.thresh(self.r[self.time],pos)\n",
        "                if thresh_met: break\n",
        "                if self.time+1==self.end:\n",
        "                    self.done=True\n",
        "                    break\n",
        "                else: self.time+=1\n",
        "                if self.partial_thresh(self.r[self.time],pos): self.l=-self.t/2\n",
        "            if pos>0: ret=r[self.time]-pos-r[self.time]*self.txcost\n",
        "            elif pos<0: ret=abs(pos)-r[self.time]-r[self.time]*self.txcost\n",
        "            ret=ret*100/self.mv\n",
        "            self.l=self.sl\n",
        "            return self.df.iloc[0:self.time+1],ret,self.done"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 60,
      "id": "488aa345-18d1-4fd3-ae6b-d29660cd424e",
      "metadata": {
        "id": "488aa345-18d1-4fd3-ae6b-d29660cd424e"
      },
      "outputs": [],
      "source": [
        "def run_episode(env,policy):\n",
        "    env.time=0\n",
        "    state=env.get_state()\n",
        "    rewards=[]\n",
        "    tot=0.0\n",
        "    done=False\n",
        "    while done==False:\n",
        "        action=policy(state)\n",
        "        state,rew,done=env.step(action)\n",
        "        tot+=rew\n",
        "        rewards+=[rew]\n",
        "        #print(state.shape[0],rew,tot)\n",
        "    return tot,rewards"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 61,
      "id": "ad23aa5a-9a5d-47f5-8b7a-a352484bb570",
      "metadata": {
        "id": "ad23aa5a-9a5d-47f5-8b7a-a352484bb570"
      },
      "outputs": [],
      "source": [
        "def always_buy(df):\n",
        "    return 1\n",
        "def always_sell(df):\n",
        "    return -1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 62,
      "id": "6ae0a35b-2357-4491-98a1-cffb6596f465",
      "metadata": {
        "id": "6ae0a35b-2357-4491-98a1-cffb6596f465"
      },
      "outputs": [],
      "source": [
        "dfrl=pd.read_csv('./dfrl.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 63,
      "id": "6d047651-0141-48aa-a845-04c52af8f226",
      "metadata": {
        "id": "6d047651-0141-48aa-a845-04c52af8f226"
      },
      "outputs": [],
      "source": [
        "# The symbols in each group are those of securities in related industries\n",
        "groups={0: [189, 167],\n",
        " 1: [269, 236, 251],\n",
        " 2: [116, 280, 90],\n",
        " 3: [137, 31, 231],\n",
        " 4: [133, 185, 23]}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 64,
      "id": "e42a0255-8a3e-4bda-9915-71756abe91b9",
      "metadata": {
        "id": "e42a0255-8a3e-4bda-9915-71756abe91b9"
      },
      "outputs": [],
      "source": [
        "# symbols=groups[0]\n",
        "symbols = dfrl['sym'].unique()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 65,
      "id": "edb408b3-7c70-4d74-aa56-b401fc4732ba",
      "metadata": {
        "id": "edb408b3-7c70-4d74-aa56-b401fc4732ba"
      },
      "outputs": [],
      "source": [
        "days=dfrl['day'].unique()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 66,
      "id": "00c69ddb-4e4e-451d-9797-8f0aeb7446b0",
      "metadata": {
        "id": "00c69ddb-4e4e-451d-9797-8f0aeb7446b0"
      },
      "outputs": [],
      "source": [
        "env=marketEnv(df=dfrl,symbol=symbols[0],day=days[10],target=0.005,stoploss=0.05)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 67,
      "id": "69cc73ed-bba8-4b7b-88cd-5bc830b8514f",
      "metadata": {
        "id": "69cc73ed-bba8-4b7b-88cd-5bc830b8514f"
      },
      "outputs": [],
      "source": [
        "ret,_=run_episode(env,always_buy)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 68,
      "id": "Vk0x07TcrBm_",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vk0x07TcrBm_",
        "outputId": "66296ec3-eb04-4981-a707-40ad46efba63"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "-4.157583752307264"
            ]
          },
          "execution_count": 68,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "ret"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "4dd0ee40",
      "metadata": {},
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": 69,
      "id": "4f8b1e35-530f-4ad3-b2f9-7a087be72907",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4f8b1e35-530f-4ad3-b2f9-7a087be72907",
        "outputId": "a3c2fcc4-99ce-47b2-ed64-b0c58e185073"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0\n",
            "-3.028761447647555 -3.028761447647555\n",
            "-0.6341861778405496 -3.6629476254881044\n",
            "1.0396762596056774 -2.623271365882427\n",
            "0.9943672842462492 -1.6289040816361777\n",
            "-1.4857681986284432 -3.114672280264621\n",
            "0.07059797568200832 -3.0440743045826126\n",
            "-0.8808627216958724 -3.924937026278485\n",
            "0.15384615196145823 -3.771090874317027\n",
            "0.5314172895636454 -3.2396735847533815\n",
            "5.327332397290714 2.0876588125373328\n",
            "2.6175343138307 4.705193126368033\n",
            "-4.319080115822993 0.3861130105450403\n",
            "3.2405141355946205 3.626627146139661\n",
            "-4.680242481021747 -1.0536153348820858\n",
            "-5.313698188299165 -6.3673135231812505\n",
            "2.9801916426971062 -3.3871218804841443\n",
            "0.891284678668181 -2.4958372018159634\n",
            "-0.7405691762558804 -3.2364063780718437\n",
            "-0.4317344981682295 -3.668140876240073\n",
            "-0.4860384316279848 -4.154179307868058\n"
          ]
        }
      ],
      "source": [
        "tot_ret=0\n",
        "print(symbols[0])\n",
        "for day in days:\n",
        "    env=marketEnv(df=dfrl,symbol=symbols[0],day=day,target=0.005,stoploss=0.05)\n",
        "    ret,_=run_episode(env,always_sell)\n",
        "    tot_ret+=ret\n",
        "    print(ret,tot_ret)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 70,
      "id": "df41f646-68b6-4bbd-ad3b-7556685dbf93",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "df41f646-68b6-4bbd-ad3b-7556685dbf93",
        "outputId": "2e255adf-0f4f-4e59-f6ce-084daf23379c"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(77, 29)"
            ]
          },
          "execution_count": 70,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "env.get_state().shape"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "-GTAR1fWtw7f",
      "metadata": {
        "id": "-GTAR1fWtw7f"
      },
      "source": []
    },
    {
      "cell_type": "markdown",
      "id": "lOIAkSwmtt22",
      "metadata": {
        "id": "lOIAkSwmtt22"
      },
      "source": [
        "## Models"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 113,
      "id": "be8d2aa0",
      "metadata": {},
      "outputs": [],
      "source": [
        "env=marketEnv(df=dfrl,symbol=symbols[0],day=days[10],target=0.005,stoploss=0.05)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 114,
      "id": "5f9fcbde",
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Open</th>\n",
              "      <th>High</th>\n",
              "      <th>Low</th>\n",
              "      <th>Close</th>\n",
              "      <th>Volume</th>\n",
              "      <th>Dividends</th>\n",
              "      <th>Open_prev</th>\n",
              "      <th>High_prev</th>\n",
              "      <th>Low_prev</th>\n",
              "      <th>Close_prev</th>\n",
              "      <th>...</th>\n",
              "      <th>BBL_5_2.0</th>\n",
              "      <th>BBM_5_2.0</th>\n",
              "      <th>BBU_5_2.0</th>\n",
              "      <th>BBB_5_2.0</th>\n",
              "      <th>BBP_5_2.0</th>\n",
              "      <th>MACD_12_26_9</th>\n",
              "      <th>MACDh_12_26_9</th>\n",
              "      <th>MACDs_12_26_9</th>\n",
              "      <th>day</th>\n",
              "      <th>sym</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>831</th>\n",
              "      <td>910.0</td>\n",
              "      <td>910.0</td>\n",
              "      <td>907.0</td>\n",
              "      <td>907.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>989.5</td>\n",
              "      <td>1000.049988</td>\n",
              "      <td>909.950012</td>\n",
              "      <td>922.049988</td>\n",
              "      <td>...</td>\n",
              "      <td>905.556341</td>\n",
              "      <td>915.809998</td>\n",
              "      <td>926.063654</td>\n",
              "      <td>2.239254</td>\n",
              "      <td>0.070397</td>\n",
              "      <td>-5.370082</td>\n",
              "      <td>-0.351575</td>\n",
              "      <td>-5.018507</td>\n",
              "      <td>10</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1 rows × 29 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "      Open   High    Low  Close  Volume  Dividends  Open_prev    High_prev  \\\n",
              "831  910.0  910.0  907.0  907.0     0.0        0.0      989.5  1000.049988   \n",
              "\n",
              "       Low_prev  Close_prev  ...   BBL_5_2.0   BBM_5_2.0   BBU_5_2.0  \\\n",
              "831  909.950012  922.049988  ...  905.556341  915.809998  926.063654   \n",
              "\n",
              "     BBB_5_2.0  BBP_5_2.0  MACD_12_26_9  MACDh_12_26_9  MACDs_12_26_9  day  \\\n",
              "831   2.239254   0.070397     -5.370082      -0.351575      -5.018507   10   \n",
              "\n",
              "     sym  \n",
              "831    0  \n",
              "\n",
              "[1 rows x 29 columns]"
            ]
          },
          "execution_count": 114,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "env.get_state()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 115,
      "id": "cc8b9780",
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "execution_count": 115,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "env.time"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 110,
      "id": "487c4aa9",
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Open</th>\n",
              "      <th>High</th>\n",
              "      <th>Low</th>\n",
              "      <th>Close</th>\n",
              "      <th>Volume</th>\n",
              "      <th>Dividends</th>\n",
              "      <th>Open_prev</th>\n",
              "      <th>High_prev</th>\n",
              "      <th>Low_prev</th>\n",
              "      <th>Close_prev</th>\n",
              "      <th>...</th>\n",
              "      <th>BBL_5_2.0</th>\n",
              "      <th>BBM_5_2.0</th>\n",
              "      <th>BBU_5_2.0</th>\n",
              "      <th>BBB_5_2.0</th>\n",
              "      <th>BBP_5_2.0</th>\n",
              "      <th>MACD_12_26_9</th>\n",
              "      <th>MACDh_12_26_9</th>\n",
              "      <th>MACDs_12_26_9</th>\n",
              "      <th>day</th>\n",
              "      <th>sym</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>831</th>\n",
              "      <td>910.000000</td>\n",
              "      <td>910.000000</td>\n",
              "      <td>907.000000</td>\n",
              "      <td>907.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>989.5</td>\n",
              "      <td>1000.049988</td>\n",
              "      <td>909.950012</td>\n",
              "      <td>922.049988</td>\n",
              "      <td>...</td>\n",
              "      <td>905.556341</td>\n",
              "      <td>915.809998</td>\n",
              "      <td>926.063654</td>\n",
              "      <td>2.239254</td>\n",
              "      <td>0.070397</td>\n",
              "      <td>-5.370082</td>\n",
              "      <td>-0.351575</td>\n",
              "      <td>-5.018507</td>\n",
              "      <td>10</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>832</th>\n",
              "      <td>907.000000</td>\n",
              "      <td>907.000000</td>\n",
              "      <td>907.000000</td>\n",
              "      <td>907.000000</td>\n",
              "      <td>19.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>989.5</td>\n",
              "      <td>1000.049988</td>\n",
              "      <td>909.950012</td>\n",
              "      <td>922.049988</td>\n",
              "      <td>...</td>\n",
              "      <td>901.889237</td>\n",
              "      <td>914.409998</td>\n",
              "      <td>926.930758</td>\n",
              "      <td>2.738544</td>\n",
              "      <td>0.204092</td>\n",
              "      <td>-5.839757</td>\n",
              "      <td>-0.657000</td>\n",
              "      <td>-5.182757</td>\n",
              "      <td>10</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>833</th>\n",
              "      <td>907.099976</td>\n",
              "      <td>907.099976</td>\n",
              "      <td>907.099976</td>\n",
              "      <td>907.099976</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>989.5</td>\n",
              "      <td>1000.049988</td>\n",
              "      <td>909.950012</td>\n",
              "      <td>922.049988</td>\n",
              "      <td>...</td>\n",
              "      <td>899.380629</td>\n",
              "      <td>912.029993</td>\n",
              "      <td>924.679356</td>\n",
              "      <td>2.773892</td>\n",
              "      <td>0.305128</td>\n",
              "      <td>-6.133210</td>\n",
              "      <td>-0.760363</td>\n",
              "      <td>-5.372848</td>\n",
              "      <td>10</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>834</th>\n",
              "      <td>907.099976</td>\n",
              "      <td>907.099976</td>\n",
              "      <td>907.099976</td>\n",
              "      <td>907.099976</td>\n",
              "      <td>20.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>989.5</td>\n",
              "      <td>1000.049988</td>\n",
              "      <td>909.950012</td>\n",
              "      <td>922.049988</td>\n",
              "      <td>...</td>\n",
              "      <td>898.049655</td>\n",
              "      <td>910.049988</td>\n",
              "      <td>922.050321</td>\n",
              "      <td>2.637291</td>\n",
              "      <td>0.377086</td>\n",
              "      <td>-6.293230</td>\n",
              "      <td>-0.736306</td>\n",
              "      <td>-5.556924</td>\n",
              "      <td>10</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>835</th>\n",
              "      <td>909.799988</td>\n",
              "      <td>909.799988</td>\n",
              "      <td>909.799988</td>\n",
              "      <td>909.799988</td>\n",
              "      <td>3.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>989.5</td>\n",
              "      <td>1000.049988</td>\n",
              "      <td>909.950012</td>\n",
              "      <td>922.049988</td>\n",
              "      <td>...</td>\n",
              "      <td>905.398171</td>\n",
              "      <td>907.599988</td>\n",
              "      <td>909.801804</td>\n",
              "      <td>0.485195</td>\n",
              "      <td>0.999587</td>\n",
              "      <td>-6.131498</td>\n",
              "      <td>-0.459659</td>\n",
              "      <td>-5.671839</td>\n",
              "      <td>10</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>836</th>\n",
              "      <td>908.299988</td>\n",
              "      <td>908.299988</td>\n",
              "      <td>908.299988</td>\n",
              "      <td>908.299988</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>989.5</td>\n",
              "      <td>1000.049988</td>\n",
              "      <td>909.950012</td>\n",
              "      <td>922.049988</td>\n",
              "      <td>...</td>\n",
              "      <td>905.696282</td>\n",
              "      <td>907.859985</td>\n",
              "      <td>910.023689</td>\n",
              "      <td>0.476660</td>\n",
              "      <td>0.601678</td>\n",
              "      <td>-6.054569</td>\n",
              "      <td>-0.306184</td>\n",
              "      <td>-5.748385</td>\n",
              "      <td>10</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>6 rows × 29 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "           Open        High         Low       Close  Volume  Dividends  \\\n",
              "831  910.000000  910.000000  907.000000  907.000000     0.0        0.0   \n",
              "832  907.000000  907.000000  907.000000  907.000000    19.0        0.0   \n",
              "833  907.099976  907.099976  907.099976  907.099976     0.0        0.0   \n",
              "834  907.099976  907.099976  907.099976  907.099976    20.0        0.0   \n",
              "835  909.799988  909.799988  909.799988  909.799988     3.0        0.0   \n",
              "836  908.299988  908.299988  908.299988  908.299988     1.0        0.0   \n",
              "\n",
              "     Open_prev    High_prev    Low_prev  Close_prev  ...   BBL_5_2.0  \\\n",
              "831      989.5  1000.049988  909.950012  922.049988  ...  905.556341   \n",
              "832      989.5  1000.049988  909.950012  922.049988  ...  901.889237   \n",
              "833      989.5  1000.049988  909.950012  922.049988  ...  899.380629   \n",
              "834      989.5  1000.049988  909.950012  922.049988  ...  898.049655   \n",
              "835      989.5  1000.049988  909.950012  922.049988  ...  905.398171   \n",
              "836      989.5  1000.049988  909.950012  922.049988  ...  905.696282   \n",
              "\n",
              "      BBM_5_2.0   BBU_5_2.0  BBB_5_2.0  BBP_5_2.0  MACD_12_26_9  \\\n",
              "831  915.809998  926.063654   2.239254   0.070397     -5.370082   \n",
              "832  914.409998  926.930758   2.738544   0.204092     -5.839757   \n",
              "833  912.029993  924.679356   2.773892   0.305128     -6.133210   \n",
              "834  910.049988  922.050321   2.637291   0.377086     -6.293230   \n",
              "835  907.599988  909.801804   0.485195   0.999587     -6.131498   \n",
              "836  907.859985  910.023689   0.476660   0.601678     -6.054569   \n",
              "\n",
              "     MACDh_12_26_9  MACDs_12_26_9  day  sym  \n",
              "831      -0.351575      -5.018507   10    0  \n",
              "832      -0.657000      -5.182757   10    0  \n",
              "833      -0.760363      -5.372848   10    0  \n",
              "834      -0.736306      -5.556924   10    0  \n",
              "835      -0.459659      -5.671839   10    0  \n",
              "836      -0.306184      -5.748385   10    0  \n",
              "\n",
              "[6 rows x 29 columns]"
            ]
          },
          "execution_count": 110,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "env.get_state()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 104,
      "id": "4d518725",
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Open</th>\n",
              "      <th>High</th>\n",
              "      <th>Low</th>\n",
              "      <th>Close</th>\n",
              "      <th>Volume</th>\n",
              "      <th>Dividends</th>\n",
              "      <th>Open_prev</th>\n",
              "      <th>High_prev</th>\n",
              "      <th>Low_prev</th>\n",
              "      <th>Close_prev</th>\n",
              "      <th>...</th>\n",
              "      <th>BBL_5_2.0</th>\n",
              "      <th>BBM_5_2.0</th>\n",
              "      <th>BBU_5_2.0</th>\n",
              "      <th>BBB_5_2.0</th>\n",
              "      <th>BBP_5_2.0</th>\n",
              "      <th>MACD_12_26_9</th>\n",
              "      <th>MACDh_12_26_9</th>\n",
              "      <th>MACDs_12_26_9</th>\n",
              "      <th>day</th>\n",
              "      <th>sym</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>831</th>\n",
              "      <td>910.000000</td>\n",
              "      <td>910.000000</td>\n",
              "      <td>907.000000</td>\n",
              "      <td>907.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>989.5</td>\n",
              "      <td>1000.049988</td>\n",
              "      <td>909.950012</td>\n",
              "      <td>922.049988</td>\n",
              "      <td>...</td>\n",
              "      <td>905.556341</td>\n",
              "      <td>915.809998</td>\n",
              "      <td>926.063654</td>\n",
              "      <td>2.239254</td>\n",
              "      <td>0.070397</td>\n",
              "      <td>-5.370082</td>\n",
              "      <td>-0.351575</td>\n",
              "      <td>-5.018507</td>\n",
              "      <td>10</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>832</th>\n",
              "      <td>907.000000</td>\n",
              "      <td>907.000000</td>\n",
              "      <td>907.000000</td>\n",
              "      <td>907.000000</td>\n",
              "      <td>19.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>989.5</td>\n",
              "      <td>1000.049988</td>\n",
              "      <td>909.950012</td>\n",
              "      <td>922.049988</td>\n",
              "      <td>...</td>\n",
              "      <td>901.889237</td>\n",
              "      <td>914.409998</td>\n",
              "      <td>926.930758</td>\n",
              "      <td>2.738544</td>\n",
              "      <td>0.204092</td>\n",
              "      <td>-5.839757</td>\n",
              "      <td>-0.657000</td>\n",
              "      <td>-5.182757</td>\n",
              "      <td>10</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>833</th>\n",
              "      <td>907.099976</td>\n",
              "      <td>907.099976</td>\n",
              "      <td>907.099976</td>\n",
              "      <td>907.099976</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>989.5</td>\n",
              "      <td>1000.049988</td>\n",
              "      <td>909.950012</td>\n",
              "      <td>922.049988</td>\n",
              "      <td>...</td>\n",
              "      <td>899.380629</td>\n",
              "      <td>912.029993</td>\n",
              "      <td>924.679356</td>\n",
              "      <td>2.773892</td>\n",
              "      <td>0.305128</td>\n",
              "      <td>-6.133210</td>\n",
              "      <td>-0.760363</td>\n",
              "      <td>-5.372848</td>\n",
              "      <td>10</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>834</th>\n",
              "      <td>907.099976</td>\n",
              "      <td>907.099976</td>\n",
              "      <td>907.099976</td>\n",
              "      <td>907.099976</td>\n",
              "      <td>20.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>989.5</td>\n",
              "      <td>1000.049988</td>\n",
              "      <td>909.950012</td>\n",
              "      <td>922.049988</td>\n",
              "      <td>...</td>\n",
              "      <td>898.049655</td>\n",
              "      <td>910.049988</td>\n",
              "      <td>922.050321</td>\n",
              "      <td>2.637291</td>\n",
              "      <td>0.377086</td>\n",
              "      <td>-6.293230</td>\n",
              "      <td>-0.736306</td>\n",
              "      <td>-5.556924</td>\n",
              "      <td>10</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>835</th>\n",
              "      <td>909.799988</td>\n",
              "      <td>909.799988</td>\n",
              "      <td>909.799988</td>\n",
              "      <td>909.799988</td>\n",
              "      <td>3.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>989.5</td>\n",
              "      <td>1000.049988</td>\n",
              "      <td>909.950012</td>\n",
              "      <td>922.049988</td>\n",
              "      <td>...</td>\n",
              "      <td>905.398171</td>\n",
              "      <td>907.599988</td>\n",
              "      <td>909.801804</td>\n",
              "      <td>0.485195</td>\n",
              "      <td>0.999587</td>\n",
              "      <td>-6.131498</td>\n",
              "      <td>-0.459659</td>\n",
              "      <td>-5.671839</td>\n",
              "      <td>10</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>836</th>\n",
              "      <td>908.299988</td>\n",
              "      <td>908.299988</td>\n",
              "      <td>908.299988</td>\n",
              "      <td>908.299988</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>989.5</td>\n",
              "      <td>1000.049988</td>\n",
              "      <td>909.950012</td>\n",
              "      <td>922.049988</td>\n",
              "      <td>...</td>\n",
              "      <td>905.696282</td>\n",
              "      <td>907.859985</td>\n",
              "      <td>910.023689</td>\n",
              "      <td>0.476660</td>\n",
              "      <td>0.601678</td>\n",
              "      <td>-6.054569</td>\n",
              "      <td>-0.306184</td>\n",
              "      <td>-5.748385</td>\n",
              "      <td>10</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>6 rows × 29 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "           Open        High         Low       Close  Volume  Dividends  \\\n",
              "831  910.000000  910.000000  907.000000  907.000000     0.0        0.0   \n",
              "832  907.000000  907.000000  907.000000  907.000000    19.0        0.0   \n",
              "833  907.099976  907.099976  907.099976  907.099976     0.0        0.0   \n",
              "834  907.099976  907.099976  907.099976  907.099976    20.0        0.0   \n",
              "835  909.799988  909.799988  909.799988  909.799988     3.0        0.0   \n",
              "836  908.299988  908.299988  908.299988  908.299988     1.0        0.0   \n",
              "\n",
              "     Open_prev    High_prev    Low_prev  Close_prev  ...   BBL_5_2.0  \\\n",
              "831      989.5  1000.049988  909.950012  922.049988  ...  905.556341   \n",
              "832      989.5  1000.049988  909.950012  922.049988  ...  901.889237   \n",
              "833      989.5  1000.049988  909.950012  922.049988  ...  899.380629   \n",
              "834      989.5  1000.049988  909.950012  922.049988  ...  898.049655   \n",
              "835      989.5  1000.049988  909.950012  922.049988  ...  905.398171   \n",
              "836      989.5  1000.049988  909.950012  922.049988  ...  905.696282   \n",
              "\n",
              "      BBM_5_2.0   BBU_5_2.0  BBB_5_2.0  BBP_5_2.0  MACD_12_26_9  \\\n",
              "831  915.809998  926.063654   2.239254   0.070397     -5.370082   \n",
              "832  914.409998  926.930758   2.738544   0.204092     -5.839757   \n",
              "833  912.029993  924.679356   2.773892   0.305128     -6.133210   \n",
              "834  910.049988  922.050321   2.637291   0.377086     -6.293230   \n",
              "835  907.599988  909.801804   0.485195   0.999587     -6.131498   \n",
              "836  907.859985  910.023689   0.476660   0.601678     -6.054569   \n",
              "\n",
              "     MACDh_12_26_9  MACDs_12_26_9  day  sym  \n",
              "831      -0.351575      -5.018507   10    0  \n",
              "832      -0.657000      -5.182757   10    0  \n",
              "833      -0.760363      -5.372848   10    0  \n",
              "834      -0.736306      -5.556924   10    0  \n",
              "835      -0.459659      -5.671839   10    0  \n",
              "836      -0.306184      -5.748385   10    0  \n",
              "\n",
              "[6 rows x 29 columns]"
            ]
          },
          "execution_count": 104,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "env.get_state()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 72,
      "id": "957ee3af",
      "metadata": {},
      "outputs": [],
      "source": [
        "EMBEDDING_DIMS = 16\n",
        "NUM_ACTIONS = 3\n",
        "INPUT_SIZE = 29\n",
        "\n",
        "dims = [INPUT_SIZE, 64, 32, EMBEDDING_DIMS]\n",
        "\n",
        "NUM_EPOCHS = 3000\n",
        "SUCCESSOR_EPOCHS = 100\n",
        "GATHERING_WALKS = 300\n",
        "MAX_WALK_LENGTH = 100\n",
        "T = 50\n",
        "\n",
        "LEARNING_RATE = 0.005\n",
        "GAMMA = 0.8\n",
        "BATCH_SIZE = 32\n",
        "MAX_CAPACITY = 1000\n",
        "MAX_WALKS = 100\n",
        "MAX_WALK_LENGTH = 100\n",
        "TASK_EPISODES = 100\n",
        "\n",
        "TRAINING_SYMBOLS = int(len(symbols) * 0.2)\n",
        "TESTING_SYMBOLS =  len(symbols) - TRAINING_SYMBOLS\n",
        "\n",
        "\n",
        "MAX_EPSILON = 1\n",
        "MIN_EPSILON = 0.01\n",
        "EPSILON_DECAY = 0.99"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 73,
      "id": "9b71adf0",
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.\n"
          ]
        },
        {
          "data": {
            "text/plain": []
          },
          "execution_count": 73,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "wandb.init('Market RL')"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "ddf25e87",
      "metadata": {},
      "source": [
        "### Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 74,
      "id": "af56c07f",
      "metadata": {},
      "outputs": [],
      "source": [
        "class PSI(nn.Module):\n",
        "    def __init__(self, dims):\n",
        "        super().__init__()\n",
        "        layers = []\n",
        "        prev = dims[0]\n",
        "        for next in dims[1:-1]:\n",
        "            layers.append(nn.Linear(prev, next))\n",
        "            layers.append(nn.ReLU())\n",
        "            prev = next\n",
        "        layers.append(nn.Linear(prev, dims[-1]))\n",
        "\n",
        "        self.layers = nn.Sequential(*layers)\n",
        "    \n",
        "    def forward(self, state):\n",
        "        return self.layers(state)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 75,
      "id": "bd86c3c5",
      "metadata": {},
      "outputs": [],
      "source": [
        "class FQN(nn.Module):\n",
        "    def __init__(self, embedding_dim, num_actions):\n",
        "        super().__init__()\n",
        "        self.linear = nn.Linear(embedding_dim, num_actions, bias=False)\n",
        "    \n",
        "    def forward(self, successor):\n",
        "        return self.linear(successor)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "4d89a774",
      "metadata": {},
      "source": [
        "### Replay"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 76,
      "id": "616aaf95",
      "metadata": {},
      "outputs": [],
      "source": [
        "from collections import namedtuple, deque\n",
        "import random\n",
        "import numpy as np\n",
        "import torch\n",
        "\n",
        "Transition = namedtuple('Transition', ('state', 'action', 'next_state', 'reward', 'done'))\n",
        "\n",
        "def get_device():\n",
        "    return torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "class FQNReplayBuffer:\n",
        "\n",
        "    def __init__(self, max_size):\n",
        "        self.memory = deque([], max_size)\n",
        "        self.max_size = max_size\n",
        "        self.device = get_device()\n",
        "\n",
        "    def push(self, state, action, next_state, reward, done):\n",
        "        transition = Transition(state, action, next_state, reward, done)\n",
        "        self.memory.append(transition)\n",
        "    \n",
        "    def sample(self, batch_size):\n",
        "        sample_size = min(batch_size, len(self))\n",
        "        sample = random.sample(self.memory, sample_size)\n",
        "        return self.convert_to_batch(sample)\n",
        "\n",
        "    def sample_to_torch(self, batch_size):\n",
        "        sample = self.sample(batch_size)\n",
        "\n",
        "        state = sample.state\n",
        "        action = sample.action\n",
        "        next_state = sample.next_state\n",
        "        reward = sample.reward\n",
        "        done = sample.done\n",
        "\n",
        "        state = torch.from_numpy(state).float().to(self.device)\n",
        "        action = torch.tensor(action, device=self.device).long()\n",
        "        next_state = torch.from_numpy(next_state).float().to(self.device)\n",
        "        reward = torch.tensor(reward, device=self.device).float()\n",
        "        done = torch.tensor(done, device=self.device).float()\n",
        "\n",
        "        return Transition(state, action, next_state, reward, done)\n",
        "    \n",
        "    def __len__(self):\n",
        "        return len(self.memory)\n",
        "    \n",
        "    def convert_to_batch(self, transitions):\n",
        "        batch = Transition(*zip(*transitions))\n",
        "\n",
        "        # if isinstance(batch.state, tuple):\n",
        "            # print(len(batch.state))\n",
        "        np_state = np.stack(batch.state)\n",
        "        np_action = np.array(batch.action)\n",
        "        np_next_state = np.stack(batch.next_state)\n",
        "        np_reward = np.array(batch.reward)\n",
        "        np_done = np.array(batch.done)\n",
        "\n",
        "        np_batch = Transition(np_state, np_action, np_next_state, np_reward, np_done)\n",
        "\n",
        "        return np_batch\n",
        "    \n",
        "    def is_full(self):\n",
        "        return len(self.memory) == self.max_size"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 77,
      "id": "f4d73b06",
      "metadata": {},
      "outputs": [],
      "source": [
        "class PSIReplayBuffer:\n",
        "    def __init__(self, max_walks):\n",
        "        self.memory = deque([], max_walks)\n",
        "        self.max_walks = max_walks\n",
        "        self.device = get_device()\n",
        "    \n",
        "    def push(self, states):\n",
        "        states = np.stack(states)\n",
        "        actions = np.stack(actions)\n",
        "        self.memory.append(states)\n",
        "    \n",
        "    def __len__(self):\n",
        "        return len(self.memory)\n",
        "    \n",
        "    def sample(self, batch_size):\n",
        "        samples = random.sample(self.memory, batch_size)\n",
        "        return samples\n",
        "    \n",
        "    def sample_to_torch(self, batch_size):\n",
        "        samples = self.sample(batch_size)\n",
        "        samples = [torch.from_numpy(states).float().to(self.device) for states in samples]\n",
        "\n",
        "        return samples"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "e2f1bea9",
      "metadata": {},
      "source": [
        "### Agent"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 78,
      "id": "a66bc6d9",
      "metadata": {},
      "outputs": [],
      "source": [
        "class Agent:\n",
        "    def __init__(self, env, psi: nn.Module, fqn: nn.Module, device=None, lr=2e-3, gamma=0.8, batch_size=64,\n",
        "                max_capacity=1000, max_walks=300, T=50):\n",
        "        self.env = env\n",
        "        self.psi = psi\n",
        "        self.fqn = fqn\n",
        "        self.device = device if device is not None else self.get_device()\n",
        "        self.lr = lr\n",
        "        self.gamma = gamma\n",
        "        self.batch_size = batch_size\n",
        "        self.T = T\n",
        "        self.psi_memory = PSIReplayBuffer(max_walks)\n",
        "        self.fqn_memory = FQNReplayBuffer(max_capacity)\n",
        "        self.psi_optimizer = torch.optim.Adam(self.psi.parameters(), lr=lr)\n",
        "        self.fqn_optimizer = torch.optim.Adam(self.fqn.parameters(), lr=lr)\n",
        "    \n",
        "    def get_device(self):\n",
        "        return torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "    \n",
        "    def train_psi(self, num_walks):\n",
        "        walks = self.psi_memory.sample(num_walks)\n",
        "        objectives = []\n",
        "        for states in walks:\n",
        "            objective = self.calculate_psi_objective(states)\n",
        "            objectives.append(objective)\n",
        "        \n",
        "        loss = -1 * torch.stack(objectives).mean() # Multiplied by -1 as we need to maximize the objective\n",
        "\n",
        "        self.psi_optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        self.psi_optimizer.step()\n",
        "\n",
        "        return loss\n",
        "    \n",
        "    def calculate_psi_objective(self, states) -> torch.Tensor:\n",
        "        num_states = len(log_successor_matrix)\n",
        "\n",
        "        successors = self.psi(states)\n",
        "        successor_matrix = torch.matmul(successors, successors.t())\n",
        "        log_successor_matrix = F.logsigmoid(successor_matrix)\n",
        "        selector = torch.triu(torch.ones(num_states, num_states), 1)\n",
        "        selector = torch.tril(selector, self.T)\n",
        "        log_successor_matrix = log_successor_matrix.mul(selector)\n",
        "\n",
        "        log_discount_matrix = torch.arange(num_states, device=self.device).repeat(num_states, 1)\n",
        "        log_discount_matrix = log_discount_matrix - torch.arange(1, num_states + 1, device=self.device).unsqueeze(-1)\n",
        "        log_discount_matrix = torch.triu(log_discount_matrix, 1)\n",
        "        log_discount_matrix = torch.tril(log_discount_matrix, self.T)\n",
        "        log_discount_matrix = log_discount_matrix * math.log(self.gamma)\n",
        "        \n",
        "        log_discounted_successor_matrix = log_successor_matrix + log_discount_matrix\n",
        "\n",
        "        objective = log_discounted_successor_matrix.sum()\n",
        "        return objective\n",
        "    \n",
        "    def train_fqn(self):\n",
        "        state, action, reward, next_state, done = self.fqn_replay.sample_to_torch(self.batch_size)\n",
        "\n",
        "        with torch.no_grad():\n",
        "            successor_feature = self.psi(state)\n",
        "            next_successor_feature = self.psi(next_state)\n",
        "\n",
        "        with torch.no_grad():\n",
        "            next_q_values = self.fqn(next_successor_feature)\n",
        "        optimal_next_q_value, _ = next_q_values.max(1)\n",
        "        target_q_values = reward + (self.gamma * optimal_next_q_value * (1 - done))\n",
        "\n",
        "        input_q_values = self.get_q_value(successor_feature)\n",
        "        q_values = input_q_values.gather(1, action.unsqueeze(1)).squeeze()\n",
        "\n",
        "        loss = F.smooth_l1_loss(q_values, target_q_values)\n",
        "\n",
        "        self.fqn_optimizer.zero_grad(set_to_none=True)\n",
        "        loss.backward()\n",
        "        self.fqn_optimizer.step()\n",
        "\n",
        "        return loss\n",
        "    \n",
        "    def act(self, state, epsilon=0):\n",
        "        if random.random() < epsilon:\n",
        "            return np.random.randint(0, NUM_ACTIONS)\n",
        "        \n",
        "        with torch.no_grad():\n",
        "            state = torch.from_numpy(state).float().to(self.device)\n",
        "            successor_feature = self.psi(state)\n",
        "            q_values = self.fqn(successor_feature)\n",
        "            action = q_values.argmax(1).item()\n",
        "            return action\n",
        "    \n",
        "    def reset_fqn(self):\n",
        "        self.fqn = FQN(EMBEDDING_DIMS, NUM_ACTIONS)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "ec92f4ce",
      "metadata": {},
      "source": [
        "### Training"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 79,
      "id": "a23bc93d",
      "metadata": {},
      "outputs": [],
      "source": [
        "fqn_model = FQN(EMBEDDING_DIMS, NUM_ACTIONS)\n",
        "psi_model = PSI(dims)\n",
        "agent = Agent(env, psi_model, fqn_model, lr=LEARNING_RATE, gamma=GAMMA, batch_size=BATCH_SIZE, max_capacity=MAX_CAPACITY, max_walks=MAX_WALKS, T=T)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 80,
      "id": "5c91c421",
      "metadata": {},
      "outputs": [
        {
          "ename": "ValueError",
          "evalue": "all input arrays must have the same shape",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[1;32m/home/anirudh/test/marketEnvForRLprojects.ipynb Cell 43'\u001b[0m in \u001b[0;36m<cell line: 8>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     <a href='vscode-notebook-cell://wsl%2Bubuntu-20.04/home/anirudh/test/marketEnvForRLprojects.ipynb#ch0000042vscode-remote?line=22'>23</a>\u001b[0m     states\u001b[39m.\u001b[39mappend(next_state)\n\u001b[1;32m     <a href='vscode-notebook-cell://wsl%2Bubuntu-20.04/home/anirudh/test/marketEnvForRLprojects.ipynb#ch0000042vscode-remote?line=23'>24</a>\u001b[0m     state \u001b[39m=\u001b[39m next_state\n\u001b[0;32m---> <a href='vscode-notebook-cell://wsl%2Bubuntu-20.04/home/anirudh/test/marketEnvForRLprojects.ipynb#ch0000042vscode-remote?line=24'>25</a>\u001b[0m agent\u001b[39m.\u001b[39;49mpsi_memory\u001b[39m.\u001b[39;49mpush(states)\n",
            "\u001b[1;32m/home/anirudh/test/marketEnvForRLprojects.ipynb Cell 38'\u001b[0m in \u001b[0;36mPSIReplayBuffer.push\u001b[0;34m(self, states)\u001b[0m\n\u001b[1;32m      <a href='vscode-notebook-cell://wsl%2Bubuntu-20.04/home/anirudh/test/marketEnvForRLprojects.ipynb#ch0000036vscode-remote?line=6'>7</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mpush\u001b[39m(\u001b[39mself\u001b[39m, states):\n\u001b[0;32m----> <a href='vscode-notebook-cell://wsl%2Bubuntu-20.04/home/anirudh/test/marketEnvForRLprojects.ipynb#ch0000036vscode-remote?line=7'>8</a>\u001b[0m     states \u001b[39m=\u001b[39m np\u001b[39m.\u001b[39;49mstack(states)\n\u001b[1;32m      <a href='vscode-notebook-cell://wsl%2Bubuntu-20.04/home/anirudh/test/marketEnvForRLprojects.ipynb#ch0000036vscode-remote?line=8'>9</a>\u001b[0m     actions \u001b[39m=\u001b[39m np\u001b[39m.\u001b[39mstack(actions)\n\u001b[1;32m     <a href='vscode-notebook-cell://wsl%2Bubuntu-20.04/home/anirudh/test/marketEnvForRLprojects.ipynb#ch0000036vscode-remote?line=9'>10</a>\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmemory\u001b[39m.\u001b[39mappend(states)\n",
            "File \u001b[0;32m<__array_function__ internals>:5\u001b[0m, in \u001b[0;36mstack\u001b[0;34m(*args, **kwargs)\u001b[0m\n",
            "File \u001b[0;32m~/.anaconda/envs/meta-rl/lib/python3.9/site-packages/numpy/core/shape_base.py:426\u001b[0m, in \u001b[0;36mstack\u001b[0;34m(arrays, axis, out)\u001b[0m\n\u001b[1;32m    <a href='file:///home/anirudh/.anaconda/envs/meta-rl/lib/python3.9/site-packages/numpy/core/shape_base.py?line=423'>424</a>\u001b[0m shapes \u001b[39m=\u001b[39m {arr\u001b[39m.\u001b[39mshape \u001b[39mfor\u001b[39;00m arr \u001b[39min\u001b[39;00m arrays}\n\u001b[1;32m    <a href='file:///home/anirudh/.anaconda/envs/meta-rl/lib/python3.9/site-packages/numpy/core/shape_base.py?line=424'>425</a>\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39m(shapes) \u001b[39m!=\u001b[39m \u001b[39m1\u001b[39m:\n\u001b[0;32m--> <a href='file:///home/anirudh/.anaconda/envs/meta-rl/lib/python3.9/site-packages/numpy/core/shape_base.py?line=425'>426</a>\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\u001b[39m'\u001b[39m\u001b[39mall input arrays must have the same shape\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[1;32m    <a href='file:///home/anirudh/.anaconda/envs/meta-rl/lib/python3.9/site-packages/numpy/core/shape_base.py?line=427'>428</a>\u001b[0m result_ndim \u001b[39m=\u001b[39m arrays[\u001b[39m0\u001b[39m]\u001b[39m.\u001b[39mndim \u001b[39m+\u001b[39m \u001b[39m1\u001b[39m\n\u001b[1;32m    <a href='file:///home/anirudh/.anaconda/envs/meta-rl/lib/python3.9/site-packages/numpy/core/shape_base.py?line=428'>429</a>\u001b[0m axis \u001b[39m=\u001b[39m normalize_axis_index(axis, result_ndim)\n",
            "\u001b[0;31mValueError\u001b[0m: all input arrays must have the same shape"
          ]
        }
      ],
      "source": [
        "# Initial data gathering\n",
        "\n",
        "NUM_DAYS = 10\n",
        "NUM_SYMBOLS = MAX_WALKS // NUM_DAYS\n",
        "SAMPLED_SYMBOLS = random.sample(list(symbols[:TRAINING_SYMBOLS]), NUM_SYMBOLS)\n",
        "\n",
        "\n",
        "for symbol in SAMPLED_SYMBOLS:\n",
        "    for day in days[:NUM_DAYS]:\n",
        "\n",
        "        # for episode in range(TASK_EPISODES):\n",
        "        env=marketEnv(df=dfrl,symbol=symbol,day=day,target=0.005,stoploss=0.05)\n",
        "\n",
        "        states = []\n",
        "\n",
        "        done = False\n",
        "        state = env.get_state().to_numpy()\n",
        "        while not done:\n",
        "            action = agent.act(state, 1) # Complete random action\n",
        "            next_state, reward, done = env.step(action)\n",
        "            next_state = next_state.to_numpy()\n",
        "            agent.fqn_memory.push(state, action, next_state, reward, done)\n",
        "            states.append(next_state)\n",
        "            state = next_state\n",
        "        agent.psi_memory.push(states)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 86,
      "id": "c2ff053a",
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(2, 29)"
            ]
          },
          "execution_count": 86,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "states[0].shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "12c307fc",
      "metadata": {},
      "outputs": [],
      "source": [
        "for epoch in range(NUM_EPOCHS):\n",
        "\n",
        "    # Train FQN\n",
        "\n",
        "    cummalative_reward = 0\n",
        "    \n",
        "    for i in range(TRAINING_SYMBOLS): # 4 symbols for training and 1 symbol for testing\n",
        "        for day in days:\n",
        "\n",
        "            average_episode_reward = 0\n",
        "\n",
        "            for episode in range(TASK_EPISODES):\n",
        "                env = marketEnv(df=dfrl,symbol=symbol,day=day,target=0.005,stoploss=0.05)\n",
        "                current_epsilon = MAX_EPSILON - MIN_EPSILON\n",
        "\n",
        "                state = env.get_state().to_numpy()\n",
        "                states = []\n",
        "                done = False\n",
        "                total_reward = 0\n",
        "\n",
        "                episode_fqn_loss = []\n",
        "\n",
        "                while not done:\n",
        "                    action = agent.act(state, current_epsilon + MIN_EPSILON)\n",
        "                    next_state, reward, done = env.step(action)\n",
        "                    total_reward += reward\n",
        "                    next_state = next_state.to_numpy()\n",
        "                    agent.fqn_memory.push(state, action, next_state, reward, done)\n",
        "                    states.append(next_state)\n",
        "                    state = next_state\n",
        "                    \n",
        "                    # Optimization\n",
        "                    iteration_fqn_loss = agent.train_fqn()\n",
        "                    episode_fqn_loss.append(iteration_fqn_loss)\n",
        "\n",
        "                agent.psi_memory.push(states)\n",
        "                episode_fqn_loss = torch.stack(episode_fqn_loss).mean()\n",
        "                average_episode_reward += total_reward\n",
        "                # Log episode statistics\n",
        "                wandb.log({\n",
        "                    'Episode FQN Loss': episode_fqn_loss.item(),\n",
        "                    'Episode Total Reward': total_reward,\n",
        "                    'Episode': episode+1\n",
        "                })\n",
        "            \n",
        "            average_episode_reward /= TASK_EPISODES\n",
        "            cummalative_reward += average_episode_reward\n",
        "\n",
        "    NUM_TASKS = TRAINING_SYMBOLS * len(days)\n",
        "    cummalative_reward /= NUM_TASKS\n",
        "\n",
        "\n",
        "    # Gathering walks\n",
        "\n",
        "    NUM_DAYS = 10\n",
        "    NUM_SYMBOLS = MAX_WALKS // NUM_DAYS\n",
        "    SAMPLED_SYMBOLS = random.sample(list(symbols[:TRAINING_SYMBOLS]), NUM_SYMBOLS)\n",
        "\n",
        "\n",
        "    for symbol in SAMPLED_SYMBOLS:\n",
        "        for day in days[:NUM_DAYS]:\n",
        "\n",
        "            # for episode in range(TASK_EPISODES):\n",
        "            env=marketEnv(df=dfrl,symbol=symbol,day=day,target=0.005,stoploss=0.05)\n",
        "\n",
        "            states = []\n",
        "\n",
        "            done = False\n",
        "            state = env.get_state().to_numpy()\n",
        "            while not done:\n",
        "                action = agent.act(state, 0) # Complete exploitation\n",
        "                next_state, reward, done = env.step(action)\n",
        "                next_state = next_state.to_numpy()\n",
        "                # agent.fqn_memory.push(state, action, next_state, reward, done)\n",
        "                states.append(next_state)\n",
        "                state = next_state\n",
        "            agent.psi_memory.push(states)\n",
        "\n",
        "\n",
        "\n",
        "    # Train PSI\n",
        "    for _ in range(SUCCESSOR_EPOCHS):\n",
        "        psi_loss = agent.train_psi(MAX_WALKS)    \n",
        "\n",
        "    # Logging epoch statistics\n",
        "    wandb.log({\n",
        "        'Epoch': epoch+1,\n",
        "        'Cummalative Reward': cummalative_reward,\n",
        "        'PSI Loss': psi_loss,\n",
        "        'FQN Loss': episode_fqn_loss\n",
        "    })"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "5bd4a0d2",
      "metadata": {},
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "name": "marketEnvForRLprojects.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.11"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}