Create_Data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/koriavinash1/anaconda2/lib/python2.7/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import os, sys, shutil\n",
    "import scipy.ndimage as snd\n",
    "import h5py\n",
    "import SimpleITK as sitk\n",
    "from shutil import copy\n",
    "import nibabel as nib\n",
    "import skimage.morphology as morph\n",
    "from skimage.feature import canny\n",
    "from tqdm import tqdm\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def imshow(*args,**kwargs):\n",
    "    \"\"\" Handy function to show multiple plots in on row, possibly with different cmaps and titles\n",
    "    Usage: \n",
    "    imshow(img1, title=\"myPlot\")\n",
    "    imshow(img1,img2, title=['title1','title2'])\n",
    "    imshow(img1,img2, cmap='hot')\n",
    "    imshow(img1,img2,cmap=['gray','Blues']) \"\"\"\n",
    "    cmap = kwargs.get('cmap', 'gray')\n",
    "    title= kwargs.get('title','')\n",
    "    if len(args)==0:\n",
    "        raise ValueError(\"No images given to imshow\")\n",
    "    elif len(args)==1:\n",
    "        plt.title(title)\n",
    "        plt.imshow(args[0], interpolation='none')\n",
    "    else:\n",
    "        n=len(args)\n",
    "        if type(cmap)==str:\n",
    "            cmap = [cmap]*n\n",
    "        if type(title)==str:\n",
    "            title= [title]*n\n",
    "        plt.figure(figsize=(n*5,10))\n",
    "        for i in range(n):\n",
    "            plt.subplot(1,n,i+1)\n",
    "            plt.title(title[i])\n",
    "            plt.imshow(args[i], cmap[i])\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "selem = morph.disk(2)\n",
    "def getWeightMap(label):\n",
    "    label = np.argmax(label, axis=3)[0]\n",
    "    edge = np.float32(morph.binary_dilation(canny(np.float32(label)), selem))\n",
    "    weight_map = np.zeros(label.shape)\n",
    "    weight_map[np.where(label>0)] = 7\n",
    "    weight_map = weight_map + 1\n",
    "    weight_map[np.where(edge==1.0)] = 25\n",
    "#     weight_map[np.where(label == 2.0)] = 15\n",
    "    return np.uint8(weight_map[None,:,:])\n",
    "\n",
    "def downSampleImage(image):\n",
    "    return np.float64(snd.interpolation.zoom(image, 0.5))\n",
    "\n",
    "def loadDicomVolume(file_path, itk_image):\n",
    "    reader = sitk.ImageSeriesReader()\n",
    "    reader.SetOutputPixelType(sitk.sitkFloat32)\n",
    "    dicom_names = reader.GetGDCMSeriesFileNames(file_path)\n",
    "    reader.SetFileNames(dicom_names)\n",
    "    itk_image = reader.Execute()\n",
    "    image_vol = sitk.GetArrayFromImage(self.itk_image)\n",
    "    image_vol = np.transpose(image_vol,(1,2,0))\n",
    "    return np.float32(image_vol)\n",
    "\n",
    "def oneHot(targets,n_class = 6):\n",
    "    axis = targets.ndim\n",
    "    ret_val = (np.arange(n_class) == np.expand_dims(targets,axis = axis)).astype(int)\n",
    "    return ret_val\n",
    "\n",
    "def histogram_equalization(arr):\n",
    "    nbr_bins = 256\n",
    "    imhist, bins = np.histogram(arr.flatten(), nbr_bins, normed = True)\n",
    "    cdf = imhist.cumsum()\n",
    "    cdf = 255 * cdf / cdf[-1]\n",
    "    original_shape = arr.shape\n",
    "    arr = np.interp(arr.flatten(), bins[:-1], cdf)\n",
    "    out_arr = arr.reshape(original_shape)\n",
    "    return out_arr\n",
    "\n",
    "def normalize(x):\n",
    "    x = np.float32(x)\n",
    "    min_val = np.min(x)\n",
    "    max_val = np.max(x)\n",
    "    ret_val = (x - min_val) / (max_val - min_val)\n",
    "    return ret_val\n",
    "\n",
    "def downSample1(slc):\n",
    "    return snd.interpolation.zoom(slc,0.5)\n",
    "\n",
    "def makeLablel(lbl, num_class = 3):\n",
    "    if num_class == 2:\n",
    "        lbl[lbl==2] = 1\n",
    "    \n",
    "    lbl = oneHot(lbl,num_class)\n",
    "    return np.uint8(lbl[None,:,:])\n",
    "\n",
    "def get_z_minmaxforbrain(lbl):\n",
    "    lbl[lbl==2] = 1\n",
    "    maxes = np.max(lbl,axis =(1,2))\n",
    "    nonzero_maxes = np.nonzero(maxes)[0]\n",
    "    mn, mx = nonzero_maxes[0] - 10, nonzero_maxes[-1] + 10\n",
    "    if mn < 0:\n",
    "        mn = 0\n",
    "    if mx >= lbl.shape[0]:\n",
    "        mx = lbl.shape[0]-1\n",
    "    return mn, mx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1792 ['.', 'raw_data', 'pac2018_data', 'PAC2018_0013.nii']\n"
     ]
    }
   ],
   "source": [
    "root = './raw_data/pac2018_data/'\n",
    "dest = './processed_data/hdf5_file/'\n",
    "\n",
    "if not os.path.exists(dest): os.makedirs(dest)\n",
    "else: \n",
    "    shutil.rmtree(dest)\n",
    "    os.makedirs(dest)\n",
    "    \n",
    "vols = []\n",
    "\n",
    "for f in next(os.walk(root))[2]:\n",
    "    vols.append(os.path.join(root, f))\n",
    "        \n",
    "vols.sort()\n",
    "print len(vols), vols[10].split(\"/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1792,)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# label paths\n",
    "labels = pd.read_csv('./raw_data/PAC2018_Covariates_Upload.csv')['Label'].as_matrix()[: len(vols)]\n",
    "age = pd.read_csv('./raw_data/PAC2018_Covariates_Upload.csv')['Age'].as_matrix()[: len(vols)]\n",
    "gender = pd.read_csv('./raw_data/PAC2018_Covariates_Upload.csv')['Gender'].as_matrix()[: len(vols)]\n",
    "tiv = pd.read_csv('./raw_data/PAC2018_Covariates_Upload.csv')['TIV'].as_matrix()[: len(vols)]\n",
    "labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1792/1792 [14:02<00:00,  2.13it/s]\n"
     ]
    }
   ],
   "source": [
    "for vol_path, label, a, g, t in tqdm(zip(vols,labels, age, gender, tiv)):\n",
    "#     print(\"working on : \" + vol_path)\n",
    "    vol_img = sitk.ReadImage(vol_path)\n",
    "    vol = sitk.GetArrayFromImage(vol_img)\n",
    "    \n",
    "    vol = histogram_equalization(vol)\n",
    "    vol = normalize(vol)\n",
    "    \n",
    "    vol = np.swapaxes(vol, 1, 2)\n",
    "    \n",
    "#     imshow(vol[:,:,72])\n",
    "    \n",
    "    dest_path = os.path.join(dest, vol_path.split(\"/\")[3][:-4] +'.hdf5')\n",
    "    hp = h5py.File(dest_path,'w')\n",
    "    hp.create_dataset('volume', data=vol)\n",
    "    hp.create_dataset('label', data=[label])\n",
    "    hp.create_dataset('gender', data=[g])\n",
    "    hp.create_dataset('tiv', data=[t])\n",
    "    hp.create_dataset('age', data=[a])\n",
    "    hp.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array([1, 2, 3]), array([1., 2.]))\n",
      "354 230 154\n",
      "465 357 232\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('./raw_data/PAC_info_sheet.csv')\n",
    "print (np.unique(df['Scanner'].as_matrix()), np.unique(df['Gender'].as_matrix()))\n",
    "scanner_1_male_df = df[(df['Scanner']==1)*df['Gender']==1.]\n",
    "scanner_2_male_df = df[(df['Scanner']==2)*df['Gender']==1.]\n",
    "scanner_3_male_df = df[(df['Scanner']==3)*df['Gender']==1.]\n",
    "\n",
    "scanner_1_female_df = df[(df['Scanner']==1)*df['Gender']==2.]\n",
    "scanner_2_female_df = df[(df['Scanner']==2)*df['Gender']==2.]\n",
    "scanner_3_female_df = df[(df['Scanner']==3)*df['Gender']==2.]\n",
    "\n",
    "print len(scanner_1_male_df), len(scanner_2_male_df), len(scanner_3_male_df)\n",
    "print len(scanner_1_female_df), len(scanner_2_female_df), len(scanner_3_female_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total number of samples: 465\n",
      "Training Set Done!!\n",
      "Validation Set Done!!\n",
      "Test Set Done!!\n",
      "Data Splitting Done...\n"
     ]
    }
   ],
   "source": [
    "src_path = './processed_data/hdf5_file'\n",
    "dest_path = './processed_data/'\n",
    "\n",
    "def generate_train_validate_test_set(df, name):\n",
    "    \"\"\"\n",
    "    Split the data into 70:15:15 for train-validate-test set\n",
    "    arg: path: input data path\n",
    "    \n",
    "    generates CSV file with slice id and corrosponding bool \n",
    "    value for train, test and validation \n",
    "    \"\"\"\n",
    "    \n",
    "    \n",
    "    SPLIT_TRAIN = 0.7\n",
    "    SPLIT_VALID = 0.15    \n",
    "      \n",
    "    train_files = df['PAC_ID'].as_matrix()\n",
    "    labels = df['Label'].as_matrix()\n",
    "    total_samples = len(train_files)\n",
    "    print \"total number of samples: {}\".format(total_samples)\n",
    "    \n",
    "    index = np.random.randint(0, total_samples, size = total_samples)\n",
    "    train_files = train_files[index]\n",
    "    labels = labels[index]\n",
    "    \n",
    "    train_vols = train_files[0:int(SPLIT_TRAIN*total_samples)]\n",
    "    valid_vols = train_files[int(SPLIT_TRAIN*total_samples): int((SPLIT_TRAIN+SPLIT_VALID)*total_samples)]\n",
    "    test_vols = train_files[int((SPLIT_TRAIN+SPLIT_VALID)*total_samples):]\n",
    "    \n",
    "    train_labels = labels[0:int(SPLIT_TRAIN*total_samples)]\n",
    "    valid_labels = labels[int(SPLIT_TRAIN*total_samples): int((SPLIT_TRAIN+SPLIT_VALID)*total_samples)]\n",
    "    test_labels = labels[int((SPLIT_TRAIN+SPLIT_VALID)*total_samples):]\n",
    "    \n",
    "    vols_path, train, valid, test, labels = [], [], [], [], [] # to save ids and corrosponding bool values\n",
    "    \n",
    "    for vol, label in zip(train_vols, train_labels):\n",
    "        folder_path = os.path.join(src_path, vol+'.hdf5')    \n",
    "        vols_path.append('.' + folder_path)\n",
    "        train.append(True)\n",
    "        valid.append(False)\n",
    "        test.append(False)\n",
    "        labels.append(label)\n",
    "    print \"Training Set Done!!\"\n",
    "    \n",
    "    for vol, label in zip(valid_vols, valid_labels):\n",
    "        folder_path = os.path.join(src_path, vol+'.hdf5')            \n",
    "        vols_path.append('.' + folder_path)\n",
    "        train.append(False)\n",
    "        valid.append(True)\n",
    "        test.append(False)\n",
    "        labels.append(label)\n",
    "    print \"Validation Set Done!!\"\n",
    "\n",
    "    for vol, label in zip(test_vols, test_labels):\n",
    "        folder_path = os.path.join(src_path, vol+'hdf5')            \n",
    "        vols_path.append('.' + folder_path)\n",
    "        train.append(False)\n",
    "        valid.append(False)\n",
    "        test.append(True)\n",
    "        labels.append(label)\n",
    "    print \"Test Set Done!!\"\n",
    "    \n",
    "    data = pd.DataFrame()\n",
    "    data['Volume_Path'] = vols_path\n",
    "    data['Labels'] = labels\n",
    "    data['Training'] = train\n",
    "    data['Testing'] = test\n",
    "    data['Validation'] = valid\n",
    "    data.to_csv(os.path.join(dest_path, name + 'train_test_split.csv'), index=False)\n",
    "    print \"Data Splitting Done...\"\n",
    "    \n",
    "generate_train_validate_test_set(scanner_1_female_df, 'scanner_1_female_')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_train_validate_test_set(src_path, labels, dest_path):\n",
    "    \"\"\"\n",
    "    Split the data into 70:15:15 for train-validate-test set\n",
    "    arg: path: input data path\n",
    "    \n",
    "    generates CSV file with slice id and corrosponding bool \n",
    "    value for train, test and validation \n",
    "    \"\"\"\n",
    "    \n",
    "    SPLIT_TRAIN = 0.7\n",
    "    SPLIT_VALID = 0.15    \n",
    "      \n",
    "    train_files = np.array(next(os.walk(src_path))[2])\n",
    "    total_samples = len(train_files)\n",
    "    print \"total number of samples: {}\".format(total_samples)\n",
    "    \n",
    "    index = np.random.randint(0, total_samples, size = total_samples)\n",
    "    train_files = train_files[index]\n",
    "    labels = labels[index]\n",
    "    \n",
    "    train_vols = train_files[0:int(SPLIT_TRAIN*total_samples)]\n",
    "    valid_vols = train_files[int(SPLIT_TRAIN*total_samples): int((SPLIT_TRAIN+SPLIT_VALID)*total_samples)]\n",
    "    test_vols = train_files[int((SPLIT_TRAIN+SPLIT_VALID)*total_samples):]\n",
    "    \n",
    "    train_labels = labels[0:int(SPLIT_TRAIN*total_samples)]\n",
    "    valid_labels = labels[int(SPLIT_TRAIN*total_samples): int((SPLIT_TRAIN+SPLIT_VALID)*total_samples)]\n",
    "    test_labels = labels[int((SPLIT_TRAIN+SPLIT_VALID)*total_samples):]\n",
    "    \n",
    "    vols_path, train, valid, test, labels = [], [], [], [], [] # to save ids and corrosponding bool values\n",
    "    \n",
    "    for vol, label in zip(train_vols, train_labels):\n",
    "        folder_path = os.path.join(src_path, vol)    \n",
    "        vols_path.append('.' + folder_path)\n",
    "        train.append(True)\n",
    "        valid.append(False)\n",
    "        test.append(False)\n",
    "        labels.append(label)\n",
    "    print \"Training Set Done!!\"\n",
    "    \n",
    "    for vol, label in zip(valid_vols, valid_labels):\n",
    "        folder_path = os.path.join(src_path, vol)            \n",
    "        vols_path.append('.' + folder_path)\n",
    "        train.append(False)\n",
    "        valid.append(True)\n",
    "        test.append(False)\n",
    "        labels.append(label)\n",
    "    print \"Validation Set Done!!\"\n",
    "\n",
    "    for vol, label in zip(test_vols, test_labels):\n",
    "        folder_path = os.path.join(src_path, vol)            \n",
    "        vols_path.append('.' + folder_path)\n",
    "        train.append(False)\n",
    "        valid.append(False)\n",
    "        test.append(True)\n",
    "        labels.append(label)\n",
    "    print \"Test Set Done!!\"\n",
    "    \n",
    "    data = pd.DataFrame()\n",
    "    data['Volume Path'] = vols_path\n",
    "    data['Labels'] = labels\n",
    "    data['Training'] = train\n",
    "    data['Testing'] = test\n",
    "    data['Validation'] = valid\n",
    "    data.to_csv(os.path.join(dest_path, 'train_test_split.csv'), index=False)\n",
    "    print \"Data Splitting Done...\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total number of samples: 1792\n",
      "Training Set Done!!\n",
      "Validation Set Done!!\n",
      "Test Set Done!!\n",
      "Data Splitting Done...\n"
     ]
    }
   ],
   "source": [
    "generate_train_validate_test_set('./processed_data/hdf5_file', labels, './processed_data/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Volume Path</th>\n",
       "      <th>Labels</th>\n",
       "      <th>Training</th>\n",
       "      <th>Testing</th>\n",
       "      <th>Validation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0747.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_1948.hdf5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_1905.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0523.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_1306.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_1674.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_1386.hdf5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_2195.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0276.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0887.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0345.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_2010.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_1797.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0778.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_1038.hdf5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0917.hdf5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_1426.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0669.hdf5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_1990.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_1126.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_1316.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_1116.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0421.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_1998.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0826.hdf5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0333.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_1160.hdf5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0099.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0732.hdf5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0606.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1762</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0416.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1763</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0900.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1764</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0475.hdf5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1765</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_1550.hdf5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1766</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_1577.hdf5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1767</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_2114.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1768</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0125.hdf5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1769</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0529.hdf5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1770</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0316.hdf5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1771</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0579.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1772</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0761.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1773</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_2187.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1774</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_1743.hdf5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1775</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0927.hdf5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1776</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_1458.hdf5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1777</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0376.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1778</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0744.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1779</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_2014.hdf5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1780</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0070.hdf5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1781</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0681.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1782</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0950.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1783</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_1666.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1784</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_1874.hdf5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1785</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0159.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1786</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_1427.hdf5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1787</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0970.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1788</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0479.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1789</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0379.hdf5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1790</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_0472.hdf5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1791</th>\n",
       "      <td>../processed_data/hdf5_file/PAC2018_1695.hdf5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1792 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        Volume Path  Labels  Training  \\\n",
       "0     ../processed_data/hdf5_file/PAC2018_0747.hdf5     1.0      True   \n",
       "1     ../processed_data/hdf5_file/PAC2018_1948.hdf5     2.0      True   \n",
       "2     ../processed_data/hdf5_file/PAC2018_1905.hdf5     1.0      True   \n",
       "3     ../processed_data/hdf5_file/PAC2018_0523.hdf5     1.0      True   \n",
       "4     ../processed_data/hdf5_file/PAC2018_1306.hdf5     1.0      True   \n",
       "5     ../processed_data/hdf5_file/PAC2018_1674.hdf5     1.0      True   \n",
       "6     ../processed_data/hdf5_file/PAC2018_1386.hdf5     2.0      True   \n",
       "7     ../processed_data/hdf5_file/PAC2018_2195.hdf5     1.0      True   \n",
       "8     ../processed_data/hdf5_file/PAC2018_0276.hdf5     1.0      True   \n",
       "9     ../processed_data/hdf5_file/PAC2018_0887.hdf5     1.0      True   \n",
       "10    ../processed_data/hdf5_file/PAC2018_0345.hdf5     1.0      True   \n",
       "11    ../processed_data/hdf5_file/PAC2018_2010.hdf5     1.0      True   \n",
       "12    ../processed_data/hdf5_file/PAC2018_1797.hdf5     1.0      True   \n",
       "13    ../processed_data/hdf5_file/PAC2018_0778.hdf5     1.0      True   \n",
       "14    ../processed_data/hdf5_file/PAC2018_1038.hdf5     2.0      True   \n",
       "15    ../processed_data/hdf5_file/PAC2018_0917.hdf5     2.0      True   \n",
       "16    ../processed_data/hdf5_file/PAC2018_1426.hdf5     1.0      True   \n",
       "17    ../processed_data/hdf5_file/PAC2018_0669.hdf5     2.0      True   \n",
       "18    ../processed_data/hdf5_file/PAC2018_1990.hdf5     1.0      True   \n",
       "19    ../processed_data/hdf5_file/PAC2018_1126.hdf5     1.0      True   \n",
       "20    ../processed_data/hdf5_file/PAC2018_1316.hdf5     1.0      True   \n",
       "21    ../processed_data/hdf5_file/PAC2018_1116.hdf5     1.0      True   \n",
       "22    ../processed_data/hdf5_file/PAC2018_0421.hdf5     1.0      True   \n",
       "23    ../processed_data/hdf5_file/PAC2018_1998.hdf5     1.0      True   \n",
       "24    ../processed_data/hdf5_file/PAC2018_0826.hdf5     2.0      True   \n",
       "25    ../processed_data/hdf5_file/PAC2018_0333.hdf5     1.0      True   \n",
       "26    ../processed_data/hdf5_file/PAC2018_1160.hdf5     2.0      True   \n",
       "27    ../processed_data/hdf5_file/PAC2018_0099.hdf5     1.0      True   \n",
       "28    ../processed_data/hdf5_file/PAC2018_0732.hdf5     2.0      True   \n",
       "29    ../processed_data/hdf5_file/PAC2018_0606.hdf5     1.0      True   \n",
       "...                                             ...     ...       ...   \n",
       "1762  ../processed_data/hdf5_file/PAC2018_0416.hdf5     1.0     False   \n",
       "1763  ../processed_data/hdf5_file/PAC2018_0900.hdf5     1.0     False   \n",
       "1764  ../processed_data/hdf5_file/PAC2018_0475.hdf5     2.0     False   \n",
       "1765  ../processed_data/hdf5_file/PAC2018_1550.hdf5     2.0     False   \n",
       "1766  ../processed_data/hdf5_file/PAC2018_1577.hdf5     2.0     False   \n",
       "1767  ../processed_data/hdf5_file/PAC2018_2114.hdf5     1.0     False   \n",
       "1768  ../processed_data/hdf5_file/PAC2018_0125.hdf5     2.0     False   \n",
       "1769  ../processed_data/hdf5_file/PAC2018_0529.hdf5     2.0     False   \n",
       "1770  ../processed_data/hdf5_file/PAC2018_0316.hdf5     2.0     False   \n",
       "1771  ../processed_data/hdf5_file/PAC2018_0579.hdf5     1.0     False   \n",
       "1772  ../processed_data/hdf5_file/PAC2018_0761.hdf5     1.0     False   \n",
       "1773  ../processed_data/hdf5_file/PAC2018_2187.hdf5     1.0     False   \n",
       "1774  ../processed_data/hdf5_file/PAC2018_1743.hdf5     2.0     False   \n",
       "1775  ../processed_data/hdf5_file/PAC2018_0927.hdf5     2.0     False   \n",
       "1776  ../processed_data/hdf5_file/PAC2018_1458.hdf5     2.0     False   \n",
       "1777  ../processed_data/hdf5_file/PAC2018_0376.hdf5     1.0     False   \n",
       "1778  ../processed_data/hdf5_file/PAC2018_0744.hdf5     1.0     False   \n",
       "1779  ../processed_data/hdf5_file/PAC2018_2014.hdf5     2.0     False   \n",
       "1780  ../processed_data/hdf5_file/PAC2018_0070.hdf5     2.0     False   \n",
       "1781  ../processed_data/hdf5_file/PAC2018_0681.hdf5     1.0     False   \n",
       "1782  ../processed_data/hdf5_file/PAC2018_0950.hdf5     1.0     False   \n",
       "1783  ../processed_data/hdf5_file/PAC2018_1666.hdf5     1.0     False   \n",
       "1784  ../processed_data/hdf5_file/PAC2018_1874.hdf5     2.0     False   \n",
       "1785  ../processed_data/hdf5_file/PAC2018_0159.hdf5     1.0     False   \n",
       "1786  ../processed_data/hdf5_file/PAC2018_1427.hdf5     2.0     False   \n",
       "1787  ../processed_data/hdf5_file/PAC2018_0970.hdf5     1.0     False   \n",
       "1788  ../processed_data/hdf5_file/PAC2018_0479.hdf5     1.0     False   \n",
       "1789  ../processed_data/hdf5_file/PAC2018_0379.hdf5     2.0     False   \n",
       "1790  ../processed_data/hdf5_file/PAC2018_0472.hdf5     1.0     False   \n",
       "1791  ../processed_data/hdf5_file/PAC2018_1695.hdf5     2.0     False   \n",
       "\n",
       "      Testing  Validation  \n",
       "0       False       False  \n",
       "1       False       False  \n",
       "2       False       False  \n",
       "3       False       False  \n",
       "4       False       False  \n",
       "5       False       False  \n",
       "6       False       False  \n",
       "7       False       False  \n",
       "8       False       False  \n",
       "9       False       False  \n",
       "10      False       False  \n",
       "11      False       False  \n",
       "12      False       False  \n",
       "13      False       False  \n",
       "14      False       False  \n",
       "15      False       False  \n",
       "16      False       False  \n",
       "17      False       False  \n",
       "18      False       False  \n",
       "19      False       False  \n",
       "20      False       False  \n",
       "21      False       False  \n",
       "22      False       False  \n",
       "23      False       False  \n",
       "24      False       False  \n",
       "25      False       False  \n",
       "26      False       False  \n",
       "27      False       False  \n",
       "28      False       False  \n",
       "29      False       False  \n",
       "...       ...         ...  \n",
       "1762     True       False  \n",
       "1763     True       False  \n",
       "1764     True       False  \n",
       "1765     True       False  \n",
       "1766     True       False  \n",
       "1767     True       False  \n",
       "1768     True       False  \n",
       "1769     True       False  \n",
       "1770     True       False  \n",
       "1771     True       False  \n",
       "1772     True       False  \n",
       "1773     True       False  \n",
       "1774     True       False  \n",
       "1775     True       False  \n",
       "1776     True       False  \n",
       "1777     True       False  \n",
       "1778     True       False  \n",
       "1779     True       False  \n",
       "1780     True       False  \n",
       "1781     True       False  \n",
       "1782     True       False  \n",
       "1783     True       False  \n",
       "1784     True       False  \n",
       "1785     True       False  \n",
       "1786     True       False  \n",
       "1787     True       False  \n",
       "1788     True       False  \n",
       "1789     True       False  \n",
       "1790     True       False  \n",
       "1791     True       False  \n",
       "\n",
       "[1792 rows x 5 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('./processed_data/train_test_split.csv')\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}