Preprocess.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Unsupervised method to cluster collagen peptide masses for a range of unknown vertebrate species"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First pre-processing step reads all CSV files available in a target root from 'monopeaklists' directories only and yields a binary matrix corresponding to protein peptide masses from a set range with a set bin size that can be changed to better suit the data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Import\n",
    "\n",
    "import csv\n",
    "import os\n",
    "import glob\n",
    "import fnmatch, re\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Preprocessing\n",
    "\n",
    "R1 = 500\n",
    "R2 = 4000\n",
    "final = pd.DataFrame()\n",
    "binsize = 1\n",
    "\n",
    "\n",
    "\n",
    "regex = fnmatch.translate('*data*.csv')\n",
    "reobj = re.compile(regex)\n",
    "\n",
    "#os.walk -> collect and read all csv files required for clustering \n",
    "exclude = ['peaklists', 'relmonopeaklists'] #read only \"monopeaklists\"\n",
    "for root, dirs, files in os.walk(\".\"):\n",
    "    dirs[:] = [d for d in dirs if d not in exclude]\n",
    "    #print(dirs)\n",
    "    for file in files:\n",
    "        filename = os.path.join(root, file)\n",
    "        print(filename)\n",
    "        if file.endswith(\".csv\") and os.stat(filename).st_size > 20 and not reobj.match(file):\n",
    "            #read all available CSV files (add exclusion of files < 20 bytes/ preprocessed data files)\n",
    "            #filename = os.path.join(root, file)\n",
    "            df1 = pd.read_csv(filename)\n",
    "            print(filename)\n",
    "            print(df1)\n",
    "            i = df1['mass'].tolist()\n",
    "            #print(df1.iloc[0])\n",
    "            #print(i)\n",
    "            mass = 0\n",
    "            rangemass = []\n",
    "            \n",
    "            for item in i:\n",
    "                if R1<float(float(item))<R2: #create labels with mass ranges\n",
    "                    mass += float(item)\n",
    "                    rangemass.append(float(item))\n",
    "                    \n",
    "            RL=R1\n",
    "            num=int((R2-R1)/binsize)\n",
    "            count = 0\n",
    "            thisrange=['filename']\n",
    "            numberofcount=[filename]\n",
    "            \n",
    "            for i in range (1,num+1): #count peaks per each bin\n",
    "                RL=R1+1*(i-binsize)\n",
    "                RH=RL+binsize\n",
    "                wantrange=(RL,RH)\n",
    "                thisrange.append(wantrange)\n",
    "                for x in rangemass:\n",
    "                    if RL<x<RH:\n",
    "                        count=count+1\n",
    "                numberofcount.append(count)\n",
    "                count=0\n",
    "            df2 = pd.DataFrame([numberofcount], columns=thisrange)\n",
    "            print(df2)\n",
    "            \n",
    "            final = final.append(df2, ignore_index=True) #update dataframe -> final CSV file (binary matrix)\n",
    "            final.to_csv(\"dataexample.csv\", sep=',')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>(500, 501)</th>\n",
       "      <th>(501, 502)</th>\n",
       "      <th>(502, 503)</th>\n",
       "      <th>(503, 504)</th>\n",
       "      <th>(504, 505)</th>\n",
       "      <th>(505, 506)</th>\n",
       "      <th>(506, 507)</th>\n",
       "      <th>(507, 508)</th>\n",
       "      <th>(508, 509)</th>\n",
       "      <th>...</th>\n",
       "      <th>(3990, 3991)</th>\n",
       "      <th>(3991, 3992)</th>\n",
       "      <th>(3992, 3993)</th>\n",
       "      <th>(3993, 3994)</th>\n",
       "      <th>(3994, 3995)</th>\n",
       "      <th>(3995, 3996)</th>\n",
       "      <th>(3996, 3997)</th>\n",
       "      <th>(3997, 3998)</th>\n",
       "      <th>(3998, 3999)</th>\n",
       "      <th>(3999, 4000)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>./20131112_P136sols/monopeaklists/20131112_P13...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>./20131112_P136sols/monopeaklists/20131112_P13...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>./20131112_P136sols/monopeaklists/20131112_P13...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>./20131112_P136sols/monopeaklists/20131112_P13...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>./20131112_P136sols/monopeaklists/20131112_P13...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 3501 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            filename  (500, 501)  (501, 502)  \\\n",
       "0  ./20131112_P136sols/monopeaklists/20131112_P13...           0           0   \n",
       "1  ./20131112_P136sols/monopeaklists/20131112_P13...           0           0   \n",
       "2  ./20131112_P136sols/monopeaklists/20131112_P13...           0           0   \n",
       "3  ./20131112_P136sols/monopeaklists/20131112_P13...           0           0   \n",
       "4  ./20131112_P136sols/monopeaklists/20131112_P13...           0           0   \n",
       "\n",
       "   (502, 503)  (503, 504)  (504, 505)  (505, 506)  (506, 507)  (507, 508)  \\\n",
       "0           0           0           0           0           0           0   \n",
       "1           0           0           0           0           0           0   \n",
       "2           0           0           0           0           0           0   \n",
       "3           0           0           0           0           0           0   \n",
       "4           0           0           0           0           0           0   \n",
       "\n",
       "   (508, 509)      ...       (3990, 3991)  (3991, 3992)  (3992, 3993)  \\\n",
       "0           0      ...                  0             0             0   \n",
       "1           0      ...                  0             0             0   \n",
       "2           0      ...                  0             0             0   \n",
       "3           0      ...                  0             0             0   \n",
       "4           0      ...                  0             0             0   \n",
       "\n",
       "   (3993, 3994)  (3994, 3995)  (3995, 3996)  (3996, 3997)  (3997, 3998)  \\\n",
       "0             0             0             0             0             0   \n",
       "1             0             0             0             0             0   \n",
       "2             0             0             0             0             0   \n",
       "3             0             0             0             0             0   \n",
       "4             0             0             0             0             0   \n",
       "\n",
       "   (3998, 3999)  (3999, 4000)  \n",
       "0             0             0  \n",
       "1             0             0  \n",
       "2             0             0  \n",
       "3             0             0  \n",
       "4             0             0  \n",
       "\n",
       "[5 rows x 3501 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#data import/show\n",
    "\n",
    "data = pd.read_csv('datalabel1.csv', index_col = [0])\n",
    "data.head()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reduced1 = data.loc[:, (data != 0).any(axis=0)]\n",
    "filelabels = reduced1.iloc[:,0]\n",
    "reduced = reduced1.iloc[:,1:]\n",
    "#print(reduced)\n",
    "#reduced.shape"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}