001.Data_Prep.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Prep\n",
    "Our dataset already has an appropraite file structure with class divided within each of the folder (train/test).  \n",
    "In this document, we will just create a validation set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, shutil\n",
    "import numpy as np\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dir = 'DATA/train'\n",
    "test_dir = 'DATA/test'\n",
    "val_dir = 'DATA/validation'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check for Corrupted Images\n",
    "(modified the original code from Keras.io)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def move_files(olddir, newdir, fnames):\n",
    "    for fn in fnames:\n",
    "        shutil.move(f'{olddir}/{fn}', f'{newdir}/{fn}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DATA/train/NORMAL\\NORMAL-1070073-0002.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-1128157-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-1317636-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-1619233-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-1702319-0002.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-1934076-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-2154341-0002.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-2244273-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-2517971-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-2558319-0002.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-2801235-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-3175613-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-3190806-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-3448549-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-3532468-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-3802540-0002.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-4212770-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-4377146-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-4521403-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-471048-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-4853105-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-557197-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-5602301-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-5722320-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-5728847-0002.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-6226952-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-6271808-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-6432470-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-6433176-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-6433176-0002.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-6478227-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-6495836-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-6534324-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-6644752-0002.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-6960239-0002.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-705474-0002.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-7451061-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-7920733-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-8272933-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-8471954-0002.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-8714174-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-8799208-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-9411713-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-9498015-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-9557588-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-9602641-0001.jpeg appears corrupted\n",
      "DATA/train/NORMAL\\NORMAL-9847458-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-1332623-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-1450110-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-1634899-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-1855562-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-2047959-0004.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-2147615-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-2532094-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-2897609-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-2897609-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-3002703-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-3170240-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-3583475-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-37006-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-3705661-0005.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-3966302-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-3971909-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-4033610-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-4092125-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-4382654-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-4531690-0008.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-4719778-0003.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-4811350-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-5004245-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-5503803-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-5503803-0003.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-5555087-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-5988773-0003.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-6121529-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-6282555-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-6398913-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-6518498-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-6715985-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-6869947-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-693904-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-6952717-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-7512869-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-7721065-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-772356-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-7869417-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-828649-0007.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-8288061-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-8288061-0003.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-8313249-0005.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-8313249-0006.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-8337454-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-8488185-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-8488185-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-8488185-0004.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-8512880-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-8607121-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-8608542-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-8738712-0011.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-8814390-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-8814390-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-8941083-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-9044474-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-9282762-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-935458-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-9434229-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\BACTERIA-9496210-0004.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-1381515-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-1393486-0003.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-1725044-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-1727027-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-1748581-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-183203-0004.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-1851402-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-1906200-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-213315-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-2445498-0003.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-2532094-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-2580396-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-2752177-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-2752177-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-2800024-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-2982529-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-3566010-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-3613917-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-3627124-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-3896078-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-3971909-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-4057077-0002.jpeg appears corrupted\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DATA/train/PNEUMONIA\\VIRUS-4493675-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-4590061-0006.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-4615614-0010.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-4615614-0011.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-463487-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-4670779-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-4728998-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-4766617-0003.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-5028760-0003.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-5123904-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-5202840-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-5284146-0003.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-5410591-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-5445920-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-5875967-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-5882420-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-5882850-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-5943537-0003.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-5988773-0004.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-5988773-0005.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-5997785-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-614452-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-6871671-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-7245232-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-7654964-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-772356-0003.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-772356-0004.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-7795904-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-7849162-0003.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-7997783-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-8201141-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-8235483-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-8343272-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-8346601-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-8365091-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-8488185-0005.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-8512880-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-8607121-0002.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-8659640-0003.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-8679052-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-8690813-0003.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-8833130-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-9044719-0004.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-9286241-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-9438146-0003.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-960851-0006.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-9613951-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-9730180-0003.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-9815549-0001.jpeg appears corrupted\n",
      "DATA/train/PNEUMONIA\\VIRUS-9902199-0004.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-1056329-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-1142234-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-1463523-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-2061674-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-2344316-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-258000-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-262135-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-280992-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-3039338-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-3174339-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-3635859-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-3746644-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-4009702-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-4190128-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-4275213-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-4276353-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-4405644-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-4533169-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-5813112-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-5880719-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-6076183-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-6076183-0002.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-6207158-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-6812602-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-6828836-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-6840343-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-6844248-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-6844248-0002.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-7638941-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-820420-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-8314555-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-842024-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-8435135-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-9039335-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-9240664-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-9564688-0001.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-9564688-0002.jpeg appears corrupted\n",
      "DATA/test/PNEUMONIA\\VIRUS-9671740-0001.jpeg appears corrupted\n"
     ]
    }
   ],
   "source": [
    "for dirp in (f'{train_dir}/NORMAL', f'{train_dir}/PNEUMONIA', f'{test_dir}/NORMAL', f'{test_dir}/PNEUMONIA'):\n",
    "    for fn in os.listdir(dirp):\n",
    "        fp = os.path.join(dirp, fn)\n",
    "        try:\n",
    "            fobj = open(fp, \"rb\")\n",
    "            is_jfif = tf.compat.as_bytes(\"JFIF\") in fobj.peek(10)\n",
    "        finally:\n",
    "            fobj.close()\n",
    "\n",
    "        if not is_jfif:\n",
    "            print(f'{fp} appears corrupted')\n",
    "            try:\n",
    "                move_files(dirp, f'ARCHIVE/{dirp}', [fn])\n",
    "            except FileNotFoundError:\n",
    "                os.makedirs(f'ARCHIVE/{dirp}')\n",
    "                move_files(dirp, f'ARCHIVE/{dirp}', [fn])\n",
    "            # os.remove(fp) # remo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "217  files removed.  5053  files in train set.  586  files in test set.\n"
     ]
    }
   ],
   "source": [
    "fc = sum(len(f) for _, _, f in os.walk('ARCHIVE'))\n",
    "fc_train = sum(len(f) for _, _, f in os.walk(train_dir))\n",
    "fc_test = sum(len(f) for _, _, f in os.walk(test_dir))\n",
    "\n",
    "print(fc, ' files removed. ', fc_train, ' files in train set. ', fc_test, ' files in test set.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Subsetting Validation Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DATA/validation already exists\n",
      "DATA/validation/NORMAL already exists\n",
      "DATA/validation/PNEUMONIA already exists\n"
     ]
    }
   ],
   "source": [
    "# make necessary directories\n",
    "def makedirect(paths):\n",
    "    for path in paths:\n",
    "        try: os.mkdir(path)\n",
    "        except FileExistsError: \n",
    "            print(f'{path} already exists')\n",
    "\n",
    "makedirect([val_dir, f'{val_dir}/NORMAL', f'{val_dir}/PNEUMONIA'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pull images from training into validation sets\n",
    "normal_imgs = [fn for fn in os.listdir(f'{train_dir}/NORMAL') if fn.endswith('.jpeg')]\n",
    "pneumo_imgs = [fn for fn in os.listdir(f'{train_dir}/PNEUMONIA') if fn.endswith('.jpeg')]\n",
    "\n",
    "# randomly select 15% of it as test set and 15% as validation set\n",
    "normal_test = np.random.choice(normal_imgs, int(.3 * len(normal_imgs)), replace = False)\n",
    "pneumo_test = np.random.choice(pneumo_imgs, int(.3 * len(pneumo_imgs)), replace = False)  \n",
    "\n",
    "normal_val = normal_test[0:int(len(normal_test)/2)]\n",
    "pneumo_val = pneumo_test[0:int(len(pneumo_test)/2)]\n",
    "normal_test = normal_test[int(len(normal_test)/2):]\n",
    "pneumo_test = pneumo_test[int(len(pneumo_test)/2):]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# move files\n",
    "move_files(f'{train_dir}/NORMAL', f'{val_dir}/NORMAL', normal_val)\n",
    "move_files(f'{train_dir}/PNEUMONIA', f'{val_dir}/PNEUMONIA', pneumo_val)\n",
    "move_files(f'{train_dir}/NORMAL', f'{test_dir}/NORMAL', normal_test)\n",
    "move_files(f'{train_dir}/PNEUMONIA', f'{test_dir}/PNEUMONIA', pneumo_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3949  files in train set.  845  files in validation set.\n"
     ]
    }
   ],
   "source": [
    "fc_train = sum(len(f) for _, _, f in os.walk(train_dir))\n",
    "fc_val = sum(len(f) for _, _, f in os.walk(val_dir))\n",
    "\n",
    "print(fc_train, ' files in train set. ', fc_val, ' files in validation set.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}