cohere-hackathon.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9d6b71f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3784b930",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9081ffba",
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f00f463b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from cleantext import clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0a1a982c",
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = 'bookmarks.html'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "655ae9a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "processed = []\n",
    "with open(filename) as f:\n",
    "    for line in f.readlines():\n",
    "        line = line.replace(\"<p>\", \"\")\n",
    "        line = line.replace(\"</DL>\", \"</DL></DT>\")\n",
    "        if line.lstrip().startswith(\"<DT><A \"):\n",
    "            line = line.rstrip() + \"</DT>\\n\"\n",
    "        processed.append(line)\n",
    "filetxt = ''.join(processed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2296eba0",
   "metadata": {},
   "outputs": [],
   "source": [
    "soup = BeautifulSoup(filetxt, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4927edd1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def visit_dt(dt, parents, collector):\n",
    "    if dt.name != 'dt':\n",
    "        raise Exception(f\"Unknown node passed as dt: {dl.name}\")\n",
    "    folder = None\n",
    "    created = None\n",
    "    modified = None\n",
    "\n",
    "    for c in dt.children:\n",
    "        if c.name == 'h3':\n",
    "            folder = c.string\n",
    "            created = c['add_date'] if c.has_attr('add_date') else None\n",
    "            modified = c['last_modified'] if c.has_attr('last_modified') else None\n",
    "        elif c.name == 'dl':\n",
    "            p = [*parents, (folder, created, modified)]\n",
    "            visit_dl(c, p, collector)\n",
    "        elif c.name == 'a':\n",
    "            if not c.has_attr('href'):\n",
    "                raise Exception(c)\n",
    "            href = c['href']\n",
    "            if not c.has_attr('add_date'):\n",
    "                raise Exception(c)\n",
    "            l_created = c['add_date']\n",
    "            l_updated = c['last_modified'] if c.has_attr('last_modified') else None\n",
    "            l_text = c.string\n",
    "            collector.append((l_text, href, l_created, l_updated, parents))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1905b764",
   "metadata": {},
   "outputs": [],
   "source": [
    "def visit_dl(dl, parents, collector):\n",
    "    if dl.name != 'dl':\n",
    "        raise Exception(f\"Unknown node passed as dl: {dl.name}\")\n",
    "    for n in dl.children:\n",
    "        if n.name == None:\n",
    "            next\n",
    "        elif n.name == 'dt':\n",
    "            visit_dt(n, parents, collector)\n",
    "        elif n.name == 'dl':\n",
    "            visit_dl(dl, parents, collector)\n",
    "        else:\n",
    "            raise Exception(f\"Unknown node {n.name}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e3149c74",
   "metadata": {},
   "outputs": [],
   "source": [
    "collector = []\n",
    "visit_dl(soup.dl, [], collector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "193e9131",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3341"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(collector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ad4899aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir -p outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23e289b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "start = 0\n",
    "batch = 50\n",
    "\n",
    "for i, c in enumerate(collector):\n",
    "    if i < start:\n",
    "        continue\n",
    "    url = c[1]\n",
    "    if not url.startswith(\"http\"):\n",
    "        continue\n",
    "    try:\n",
    "        resp = requests.get(url, timeout=1.0)\n",
    "    except Exception as e:\n",
    "        print(f\"Error while requesting {url}: {e}\")\n",
    "        continue\n",
    "    \n",
    "    filename = f\"outputs/out_{i:05}.html\"\n",
    "    with open(filename, \"w\") as f:\n",
    "        f.write(resp.text)\n",
    "#         print(f\"Output to file {filename}\")\n",
    "    if i > start + batch:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "019a2963",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import itertools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "9d38e809",
   "metadata": {},
   "outputs": [],
   "source": [
    "def cleantxt(txt):\n",
    "    return clean(text=txt,\n",
    "            fix_unicode=True,\n",
    "            to_ascii=True,\n",
    "            lower=True,\n",
    "            no_line_breaks=True,\n",
    "            no_urls=False,\n",
    "            no_emails=False,\n",
    "            no_phone_numbers=False,\n",
    "            no_numbers=False,\n",
    "            no_digits=False,\n",
    "            no_currency_symbols=False,\n",
    "            no_punct=False,\n",
    "            replace_with_punct=\"\",\n",
    "            replace_with_url=\":url:\",\n",
    "            replace_with_email=\":email:\",\n",
    "            replace_with_phone_number=\"\",\n",
    "            replace_with_number=\":num:\",\n",
    "            replace_with_digit=\":digit:\",\n",
    "            replace_with_currency_symbol=\"$\",\n",
    "            lang=\"en\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9918b8ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "files = os.listdir('outputs/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e4994ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_texts(files):\n",
    "    result = []\n",
    "    for file in files:\n",
    "        filename = f\"outputs/{file}\"\n",
    "#         print(f\"Reading file {filename}\")\n",
    "        with open(filename, \"r\") as f:\n",
    "            txt = f.read()\n",
    "        soup = BeautifulSoup(txt, \"html.parser\")\n",
    "        title = soup.title.text if soup.title else \"\"\n",
    "        content = []\n",
    "        txt = cleantxt(title)\n",
    "        content.append(f\"<title>{txt}</title>\")\n",
    "\n",
    "        main = soup.find(id=['main', 'content'])\n",
    "        if not main:\n",
    "            main = soup.body\n",
    "        if not main:\n",
    "            continue\n",
    "        for c in main.descendants:\n",
    "            if c.name in ['h1', 'h2', 'h3']:\n",
    "                txt = cleantxt(c.text)\n",
    "                content.append(f\"<{c.name}>{txt}</{c.name}>\")\n",
    "            elif c.name in ['p']:\n",
    "                txt = cleantxt(c.text[0:200])\n",
    "                if len(txt) == 0:\n",
    "                    continue\n",
    "                content.append(f\"<p>{txt}</p>\")\n",
    "            if len(content) >= 10:\n",
    "                result.append(\"\\n\".join(content))\n",
    "                break\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4da820d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "9aeeeb12",
   "metadata": {},
   "source": [
    "# Identify Topics"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f95f8c8c",
   "metadata": {},
   "source": [
    "### Create Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "dd6034be",
   "metadata": {},
   "outputs": [],
   "source": [
    "api_key = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "bfbd746a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import cohere\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "d2b52d38",
   "metadata": {},
   "outputs": [],
   "source": [
    "co = cohere.Client(api_key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "19dd465d",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = 'medium'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "8296de8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "files = os.listdir('outputs/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "40b1e404",
   "metadata": {},
   "outputs": [],
   "source": [
    "batch = 16\n",
    "\n",
    "def batch_embed(inputs):\n",
    "    result = []\n",
    "    for i in range(0, len(inputs), batch):        \n",
    "        texts = inputs[i:i+batch]\n",
    "        response = co.embed(\n",
    "            texts=texts, \n",
    "            model=model,\n",
    "            truncate=\"RIGHT\"\n",
    "        )\n",
    "        result.extend(response.embeddings)\n",
    "        print(f\"batch complete: {i}\")\n",
    "        time.sleep(1.0)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d20fb18e",
   "metadata": {},
   "source": [
    "## Plot the embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b0aa664",
   "metadata": {},
   "outputs": [],
   "source": [
    "import umap\n",
    "import altair as alt\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e1ccde6",
   "metadata": {},
   "outputs": [],
   "source": [
    "reducer = umap.UMAP(n_neighbors=100)\n",
    "umap_embeds = reducer.fit_transform(embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b7988ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "keys = list(result.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f69e2280",
   "metadata": {},
   "outputs": [],
   "source": [
    "keys = [re.sub('out_0+|\\.html', '', k) for k in keys]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "019adefc",
   "metadata": {},
   "outputs": [],
   "source": [
    "items = [collector[int(k)] for k in keys]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca5fc135",
   "metadata": {},
   "outputs": [],
   "source": [
    "titles = [item[0] for item in items]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "047f2e75",
   "metadata": {},
   "outputs": [],
   "source": [
    "titles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e77e94b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(titles, columns=['titles'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9006f5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "46dcd93b",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(umap_embeds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9147279b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['x'] = umap_embeds[:,0]\n",
    "df['y'] = umap_embeds[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "913101a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "645ea308",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Plot\n",
    "chart = alt.Chart(df).mark_circle(size=60).encode(\n",
    "    x=#'x',\n",
    "    alt.X('x',\n",
    "        scale=alt.Scale(zero=False),\n",
    "        axis=alt.Axis(labels=False, ticks=False, domain=False)\n",
    "    ),\n",
    "    y=\n",
    "    alt.Y('y',\n",
    "        scale=alt.Scale(zero=False),\n",
    "        axis=alt.Axis(labels=False, ticks=False, domain=False)\n",
    "    ),\n",
    "    tooltip=['titles']\n",
    ").configure(background=\"#FDF7F0\"\n",
    ").properties(\n",
    "    width=700,\n",
    "    height=400,\n",
    "    title='Bookmarks'\n",
    ")\n",
    "\n",
    "chart.interactive()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cd249b91",
   "metadata": {},
   "source": [
    "# Plot Only using Titles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "610c1c08",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3341"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(collector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "cdc2b97f",
   "metadata": {},
   "outputs": [],
   "source": [
    "titles = [c[0] for c in collector]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "af3cef36",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3341"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(titles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aee63fb6",
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings = batch_embed(titles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e4dcb79",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f0758d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "reducer = umap.UMAP(n_neighbors=100)\n",
    "umap_embeds = reducer.fit_transform(embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fcedb7a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(titles, columns=['titles'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "44c1f839",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be844c22",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['x'] = umap_embeds[:,0]\n",
    "df['y'] = umap_embeds[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "48b19c1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Plot\n",
    "chart = alt.Chart(df).mark_circle(size=60).encode(\n",
    "    x=#'x',\n",
    "    alt.X('x',\n",
    "        scale=alt.Scale(zero=False),\n",
    "        axis=alt.Axis(labels=False, ticks=False, domain=False)\n",
    "    ),\n",
    "    y=\n",
    "    alt.Y('y',\n",
    "        scale=alt.Scale(zero=False),\n",
    "        axis=alt.Axis(labels=False, ticks=False, domain=False)\n",
    "    ),\n",
    "    tooltip=['titles']\n",
    ").configure(background=\"#FDF7F0\"\n",
    ").properties(\n",
    "    width=700,\n",
    "    height=400,\n",
    "    title='Bookmarks'\n",
    ")\n",
    "\n",
    "chart.interactive()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc46dcfb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc898690",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "46afbb5c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a74413ea",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1668a52",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "basic-ds",
   "language": "python",
   "name": "basic-ds"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}