testing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.mixture import GaussianMixture\n",
    "import lmdb\n",
    "import numpy as np\n",
    "import lmdb\n",
    "from tqdm import tqdm\n",
    "\n",
    "DB_clean_features = lmdb.open('./clean/features.lmdb/',map_size=1200*1_000_000)  \n",
    "DB_clean_id_to_filename = lmdb.open('./clean/id_to_filename.lmdb/',map_size=50*1_000_000) #50mb\n",
    "\n",
    "def int_to_bytes(x: int) -> bytes:\n",
    "    return x.to_bytes(4, 'big')\n",
    "\n",
    "def int_from_bytes(xbytes: bytes) -> int:\n",
    "    return int.from_bytes(xbytes, 'big')\n",
    "\n",
    "def get_all_data(db, size=20000):\n",
    "    with db.begin(buffers=True) as txn:\n",
    "        with txn.cursor() as curs:\n",
    "            ids = [] \n",
    "            features = [] \n",
    "            i=0\n",
    "            for data in tqdm(curs.iternext(keys=True, values=True)):\n",
    "                if i>=size:\n",
    "                    break\n",
    "                ids.append(int_from_bytes(data[0]))\n",
    "                features.append(np.frombuffer(data[1],dtype=np.float32))\n",
    "                i+=1\n",
    "            return ids, features\n",
    "\n",
    "\n",
    "def get_file_name(image_id,file_name_db):\n",
    "    with file_name_db.begin(buffers=False) as txn:\n",
    "        file_name = txn.get(int_to_bytes(image_id), default=False)\n",
    "        return file_name.decode(\"utf-8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_ids, clean_features = get_all_data(DB_clean_features, 300000)\n",
    "clean_ids, clean_features = np.array(clean_ids), np.array(clean_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gmm = GaussianMixture(n_components = 16, covariance_type = 'full')\n",
    "gmm.fit(clean_features)\n",
    "\n",
    "# import pickle \n",
    "# with open(\"./gmm.model\",\"wb\") as file:\n",
    "#     pickle.dump(gmm,file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "clean_scores = gmm.score_samples(clean_features)\n",
    "sns.histplot([el for el in clean_scores if el>0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DB_test_features = lmdb.open('./test/features.lmdb/',map_size=1200*1_000_000) \n",
    "DB_test_id_to_filename = lmdb.open('./test/id_to_filename.lmdb/',map_size=50*1_000_000)\n",
    "\n",
    "test_ids, test_features = get_all_data(DB_test_features, 300000)\n",
    "test_ids, test_features = np.array(test_ids), np.array(test_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_scores = gmm.score_samples(test_features)\n",
    "sns.histplot([el for el in test_scores if el>0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from PIL import Image\n",
    "import math\n",
    "\n",
    "def read_img_file(f):\n",
    "    img = Image.open(f)\n",
    "    if img.mode != 'RGB':\n",
    "        img = img.convert('RGB')\n",
    "    return img\n",
    "\n",
    "def plot_imgs(file_names,IMG_PATH):\n",
    "    s = math.ceil(math.sqrt(len(file_names)))\n",
    "    _, axs = plt.subplots(s, s, figsize=(12, 12))\n",
    "    axs = axs.flatten()\n",
    "    imgs = [np.array(read_img_file(IMG_PATH+el).resize((256,256))) for el in file_names]\n",
    "    for img, ax in zip(imgs, axs):\n",
    "        ax.imshow(img)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_filenames_range =  [get_file_name(int(clean_ids[idx]),DB_clean_id_to_filename) for idx,score in enumerate(clean_scores) if 1000<score<2000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_imgs(clean_filenames_range[:64],\"./clean/images/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_filenames_range =  [get_file_name(int(test_ids[idx]),DB_test_id_to_filename) for idx,score in enumerate(test_scores) if 1000<score<2000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_imgs(test_filenames_range[:64],\"./test/images/\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.9.13 (tags/v3.9.13:6de2ca5, May 17 2022, 16:36:42) [MSC v.1929 64 bit (AMD64)]"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "1d37589d4eb4d142da20bdbebb4352afaee29589507962c05c4960916c8b7369"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}