notebooks/langchain_decoded_4_indexes.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# **LangChain Decoded**"
      ],
      "metadata": {
        "id": "JQxt18IfZR40"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Getting Started"
      ],
      "metadata": {
        "id": "zgFXxUsp5pbu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Install the LangChain package\n",
        "!pip install langchain"
      ],
      "metadata": {
        "id": "fRUe2wO95xoR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Install the OpenAI package\n",
        "!pip install openai"
      ],
      "metadata": {
        "id": "T7TwICBe9yy4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Configure the API key\n",
        "import os\n",
        "\n",
        "openai_api_key = os.environ.get('OPENAI_API_KEY', 'sk-XXX')"
      ],
      "metadata": {
        "id": "I-ZUDlva6m3Y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Part 4: Indexes"
      ],
      "metadata": {
        "id": "MTkGMnWG_lyh"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Document Loaders"
      ],
      "metadata": {
        "id": "Pc8d5uA7_4u_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install unstructured tabulate pdf2image pytesseract"
      ],
      "metadata": {
        "id": "i6zYaxTl_rlU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# URL Loader\n",
        "from langchain.document_loaders import UnstructuredURLLoader\n",
        "\n",
        "urls = [\"https://alphasec.io/summarize-text-with-langchain-and-openai\"]\n",
        "loader = UnstructuredURLLoader(urls=urls)\n",
        "data = loader.load()\n",
        "print(data)"
      ],
      "metadata": {
        "id": "LiLWQ_tF_686"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install pypdf"
      ],
      "metadata": {
        "id": "U_uGRyko_Hbk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# PDF Loader\n",
        "from langchain.document_loaders import PyPDFLoader\n",
        "\n",
        "loader = PyPDFLoader(\"./data/attention-is-all-you-need.pdf\")\n",
        "pages = loader.load_and_split()\n",
        "pages[0]"
      ],
      "metadata": {
        "id": "EdpCr38O-olp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# File Directory Loader\n",
        "from langchain.document_loaders import DirectoryLoader\n",
        "\n",
        "loader = DirectoryLoader('data', glob=\"**/*.csv\")\n",
        "docs = loader.load()\n",
        "len(docs)"
      ],
      "metadata": {
        "id": "dJIv52X3EbyD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install pytube youtube-transcript-api"
      ],
      "metadata": {
        "id": "080GK3Q3Iv5_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# YouTube Transcripts Loader\n",
        "from langchain.document_loaders import YoutubeLoader\n",
        "\n",
        "loader = YoutubeLoader.from_youtube_url(\"https://www.youtube.com/watch?v=yEgHrxvLsz0\", add_video_info=True)\n",
        "data = loader.load()\n",
        "print(data)"
      ],
      "metadata": {
        "id": "B13lagzPGn1n"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install google-cloud-storage"
      ],
      "metadata": {
        "id": "nte5y7xxa9We"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Google Cloud Storage File Loader\n",
        "from langchain.document_loaders import GCSFileLoader\n",
        "\n",
        "loader = GCSFileLoader(project_name=\"langchain-gcs\", bucket=\"langchain-gcs\", blob=\"lorem-ipsum.txt\")\n",
        "data = loader.load()\n",
        "print(data)"
      ],
      "metadata": {
        "id": "kmpul4UqbXuJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Text Splitters"
      ],
      "metadata": {
        "id": "dIEPnFzw_7Rx"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Character Text Splitter\n",
        "from langchain.text_splitter import CharacterTextSplitter\n",
        "from google.colab import files\n",
        "\n",
        "uploaded = files.upload()\n",
        "filename = next(iter(uploaded))\n",
        "text = uploaded[filename].decode(\"utf-8\")\n",
        "\n",
        "text_splitter = CharacterTextSplitter(        \n",
        "    separator = \"\\n\\n\",\n",
        "    chunk_size = 1000,\n",
        "    chunk_overlap  = 200,\n",
        "    length_function = len,\n",
        ")\n",
        "\n",
        "texts = text_splitter.create_documents([text])\n",
        "print(texts[0])\n",
        "print(texts[1])\n",
        "print(texts[2])"
      ],
      "metadata": {
        "id": "WGb0P3fS__BD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Recursive Character Text Splitter\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from google.colab import files\n",
        "\n",
        "uploaded = files.upload()\n",
        "filename = next(iter(uploaded))\n",
        "text = uploaded[filename].decode(\"utf-8\")\n",
        "\n",
        "text_splitter = RecursiveCharacterTextSplitter(        \n",
        "    chunk_size = 100,\n",
        "    chunk_overlap  = 20,\n",
        "    length_function = len,\n",
        ")\n",
        "\n",
        "texts = text_splitter.create_documents([text])\n",
        "print(texts[0])\n",
        "print(texts[1])\n",
        "print(texts[2])"
      ],
      "metadata": {
        "id": "NmYW0cN0UhxP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Vector Stores"
      ],
      "metadata": {
        "id": "yF32kJni__hY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install chromadb tiktoken"
      ],
      "metadata": {
        "id": "rMtIR-tRX0Qt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Chroma Vector Store\n",
        "import os, tiktoken\n",
        "from langchain.document_loaders import TextLoader\n",
        "from langchain.text_splitter import CharacterTextSplitter\n",
        "from langchain.vectorstores import Chroma\n",
        "from langchain.embeddings.openai import OpenAIEmbeddings\n",
        "\n",
        "OPENAI_API_KEY = '' # @param {type:\"string\"}\n",
        "os.environ[\"OPENAI_API_KEY\"] = OPENAI_API_KEY\n",
        "\n",
        "from google.colab import files\n",
        "\n",
        "uploaded = files.upload()\n",
        "filename = next(iter(uploaded))\n",
        "\n",
        "loader = TextLoader(filename)\n",
        "data = loader.load()\n",
        "\n",
        "text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)\n",
        "docs = text_splitter.split_documents(data)\n",
        "\n",
        "embeddings = OpenAIEmbeddings()\n",
        "db = Chroma.from_documents(docs, embeddings)\n",
        "\n",
        "query = \"What comes after 'Vestibulum congue convallis finibus'?\"\n",
        "docs = db.similarity_search(query)\n",
        "\n",
        "print(docs[0].page_content)"
      ],
      "metadata": {
        "id": "0XUsuto_ABze"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Retrievers"
      ],
      "metadata": {
        "id": "1RpFQvgAACR6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install arxiv pymupdf"
      ],
      "metadata": {
        "id": "ZYC12MjFPQTo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Arxiv Retriever\n",
        "from langchain.retrievers import ArxivRetriever\n",
        "\n",
        "retriever = ArxivRetriever(load_max_docs=2)\n",
        "docs = retriever.get_relevant_documents(query='2203.15556')\n",
        "\n",
        "docs[0].metadata"
      ],
      "metadata": {
        "id": "mwVrbBUfAEaG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install wikipedia"
      ],
      "metadata": {
        "id": "26WMZAxdTEzs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Wikipedia Retriever\n",
        "from langchain.retrievers import WikipediaRetriever\n",
        "\n",
        "retriever = WikipediaRetriever()\n",
        "docs = retriever.get_relevant_documents(query='large language models')\n",
        "\n",
        "docs[0].metadata"
      ],
      "metadata": {
        "id": "-pY7LCZ8TG-8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install chromadb tiktoken"
      ],
      "metadata": {
        "id": "8LrQSBVaUFmX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Chroma Vector Store Retriever\n",
        "import os, tiktoken\n",
        "from langchain.document_loaders import TextLoader\n",
        "from langchain.text_splitter import CharacterTextSplitter\n",
        "from langchain.vectorstores import Chroma\n",
        "from langchain.embeddings.openai import OpenAIEmbeddings\n",
        "\n",
        "OPENAI_API_KEY = '' # @param {type:\"string\"}\n",
        "os.environ[\"OPENAI_API_KEY\"] = OPENAI_API_KEY\n",
        "\n",
        "from google.colab import files\n",
        "\n",
        "uploaded = files.upload()\n",
        "filename = next(iter(uploaded))\n",
        "\n",
        "loader = TextLoader(filename)\n",
        "data = loader.load()\n",
        "\n",
        "text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)\n",
        "docs = text_splitter.split_documents(data)\n",
        "\n",
        "embeddings = OpenAIEmbeddings()\n",
        "db = Chroma.from_documents(docs, embeddings)\n",
        "\n",
        "retriever = db.as_retriever()\n",
        "query = \"What comes after 'Vestibulum congue convallis finibus'?\"\n",
        "docs = retriever.get_relevant_documents(query)\n",
        "\n",
        "print(docs[0].page_content)"
      ],
      "metadata": {
        "id": "S0ZiBwNKSaqS"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}