ARXIV_Conformal_Prediction.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/MatthewK84/LinkedIn-Learning-Journey/blob/main/ARXIV_Conformal_Prediction.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "bfacb0ba",
      "metadata": {
        "id": "bfacb0ba"
      },
      "outputs": [],
      "source": [
        "!pip install arxiv --user"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "id": "5ad512df",
      "metadata": {
        "id": "5ad512df"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import arxiv"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "id": "81141bbc",
      "metadata": {
        "id": "81141bbc"
      },
      "outputs": [],
      "source": [
        "# i only use these if I want to remove annoying deprecation warnings from my analysis\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "id": "4d25a4b2",
      "metadata": {
        "id": "4d25a4b2"
      },
      "outputs": [],
      "source": [
        "def search_arxiv(query, max_results=10):\n",
        "\n",
        "    data = {}\n",
        "    i = 0\n",
        "\n",
        "    search = arxiv.Search(query=query, max_results=max_results)\n",
        "\n",
        "    for result in search.results():\n",
        "\n",
        "        try:\n",
        "\n",
        "            data[i] = {}\n",
        "\n",
        "            data[i]['title'] = result.title\n",
        "            data[i]['date_published'] = result.published\n",
        "            data[i]['authors'] = [a.name for a in result.authors]\n",
        "            data[i]['summary'] = result.summary\n",
        "            data[i]['url'] = result.pdf_url\n",
        "            data[i]['category'] = result.primary_category\n",
        "\n",
        "        except:\n",
        "\n",
        "            print('weird arxiv error')\n",
        "\n",
        "        # there are more fields that can be added; add as many as you need\n",
        "\n",
        "        i += 1\n",
        "\n",
        "    df = pd.DataFrame(data).T\n",
        "    df = df[['date_published', 'title', 'authors', 'summary', 'url', 'category']]\n",
        "    df['date_published'] = pd.to_datetime(df['date_published'])\n",
        "    df.sort_values('date_published', ascending=False, inplace=True)\n",
        "    df.reset_index(drop=True, inplace=True)\n",
        "    return df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "id": "92615d2a",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "92615d2a",
        "outputId": "915e22fd-688d-454d-8c17-f8762f4d0577"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "             date_published  \\\n",
              "0 2023-11-03 09:01:37+00:00   \n",
              "1 2023-11-02 17:59:30+00:00   \n",
              "2 2023-11-02 17:58:37+00:00   \n",
              "3 2023-11-02 01:56:03+00:00   \n",
              "4 2023-11-01 18:37:07+00:00   \n",
              "\n",
              "                                               title  \\\n",
              "0  Fast ellipsoidal conformal and quasi-conformal...   \n",
              "1  Conformal Policy Learning for Sensorimotor Con...   \n",
              "2  Investigating the Behavior of Diffusion Models...   \n",
              "3  From O(3) to Cubic CFT: Conformal Perturbation...   \n",
              "4  Conformalized Deep Splines for Optimal and Eff...   \n",
              "\n",
              "                                             authors  \\\n",
              "0                                  [Gary P. T. Choi]   \n",
              "1  [Huang Huang, Satvik Sharma, Antonio Loquercio...   \n",
              "2  [Daniel Rothchild, Andrew S. Rosen, Eric Taw, ...   \n",
              "3                            [Junchen Rong, Ning Su]   \n",
              "4  [Nathaniel Diamant, Ehsan Hajiramezanali, Tomm...   \n",
              "\n",
              "                                             summary  \\\n",
              "0  Surface parameterization plays a fundamental r...   \n",
              "1  This paper focuses on the problem of detecting...   \n",
              "2  We present an investigation into diffusion mod...   \n",
              "3  The Cubic CFT can be understood as the O(3) in...   \n",
              "4  Uncertainty estimation is critical in high-sta...   \n",
              "\n",
              "                                 url         category  \n",
              "0  http://arxiv.org/pdf/2311.01788v1            cs.CG  \n",
              "1  http://arxiv.org/pdf/2311.01457v1            cs.RO  \n",
              "2  http://arxiv.org/pdf/2311.01491v1  physics.chem-ph  \n",
              "3  http://arxiv.org/pdf/2311.00933v1           hep-th  \n",
              "4  http://arxiv.org/pdf/2311.00774v1            cs.LG  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-5a9259a6-e3d2-4fe1-b1b2-ce4351bddbc9\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date_published</th>\n",
              "      <th>title</th>\n",
              "      <th>authors</th>\n",
              "      <th>summary</th>\n",
              "      <th>url</th>\n",
              "      <th>category</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2023-11-03 09:01:37+00:00</td>\n",
              "      <td>Fast ellipsoidal conformal and quasi-conformal...</td>\n",
              "      <td>[Gary P. T. Choi]</td>\n",
              "      <td>Surface parameterization plays a fundamental r...</td>\n",
              "      <td>http://arxiv.org/pdf/2311.01788v1</td>\n",
              "      <td>cs.CG</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2023-11-02 17:59:30+00:00</td>\n",
              "      <td>Conformal Policy Learning for Sensorimotor Con...</td>\n",
              "      <td>[Huang Huang, Satvik Sharma, Antonio Loquercio...</td>\n",
              "      <td>This paper focuses on the problem of detecting...</td>\n",
              "      <td>http://arxiv.org/pdf/2311.01457v1</td>\n",
              "      <td>cs.RO</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2023-11-02 17:58:37+00:00</td>\n",
              "      <td>Investigating the Behavior of Diffusion Models...</td>\n",
              "      <td>[Daniel Rothchild, Andrew S. Rosen, Eric Taw, ...</td>\n",
              "      <td>We present an investigation into diffusion mod...</td>\n",
              "      <td>http://arxiv.org/pdf/2311.01491v1</td>\n",
              "      <td>physics.chem-ph</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2023-11-02 01:56:03+00:00</td>\n",
              "      <td>From O(3) to Cubic CFT: Conformal Perturbation...</td>\n",
              "      <td>[Junchen Rong, Ning Su]</td>\n",
              "      <td>The Cubic CFT can be understood as the O(3) in...</td>\n",
              "      <td>http://arxiv.org/pdf/2311.00933v1</td>\n",
              "      <td>hep-th</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2023-11-01 18:37:07+00:00</td>\n",
              "      <td>Conformalized Deep Splines for Optimal and Eff...</td>\n",
              "      <td>[Nathaniel Diamant, Ehsan Hajiramezanali, Tomm...</td>\n",
              "      <td>Uncertainty estimation is critical in high-sta...</td>\n",
              "      <td>http://arxiv.org/pdf/2311.00774v1</td>\n",
              "      <td>cs.LG</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-5a9259a6-e3d2-4fe1-b1b2-ce4351bddbc9')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-5a9259a6-e3d2-4fe1-b1b2-ce4351bddbc9 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-5a9259a6-e3d2-4fe1-b1b2-ce4351bddbc9');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-33df3070-3da5-4c7f-a240-3ee1381bceb0\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-33df3070-3da5-4c7f-a240-3ee1381bceb0')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-33df3070-3da5-4c7f-a240-3ee1381bceb0 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ],
      "source": [
        "# at 1000 articles, it takes about 30 seconds\n",
        "# at 10000 articles, it takes a while; put laptop down and walk away\n",
        "# i haven't tried beyond 10000 yet\n",
        "\n",
        "# at the moment, this is finicky beyond 10000; needs debugged or something\n",
        "# could also be a problem with the arxiv library as it seems to be inside the search\n",
        "\n",
        "# so far, i haven't noticed obvious throttling, but possibly throttling is causing the issue\n",
        "\n",
        "query = 'Conformal Prediction'\n",
        "\n",
        "max_results = 5100\n",
        "\n",
        "df = search_arxiv(query, max_results)\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "26b9e534",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "26b9e534",
        "outputId": "650dc54e-bdb6-47d4-8efe-bf0ce5124879"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(5100, 6)"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ],
      "source": [
        "df.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "46b3d673",
      "metadata": {
        "id": "46b3d673"
      },
      "outputs": [],
      "source": [
        "outfile = '/content/sample_data/conformal_prediction_data.csv'\n",
        "\n",
        "df.to_csv(outfile, index=False)"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.16"
    },
    "colab": {
      "provenance": [],
      "include_colab_link": true
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}