MakeTFRecords.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convert Numpy Arrays to TF Records\n",
    "Save the CullPDB protein dataset as TF records to make importing them to models easier.\n",
    "This notebook is a walkthrough of the file *make_tfrecords.py*. I find it is\n",
    "easiest to understand what a data pipeline is doing if I can visualize what the data \n",
    "looks like as it moves through that pipeline."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "\n",
    "# define tf.train.Features we need\n",
    "def _int64_feature(value):\n",
    "    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n",
    "\n",
    "def _floats_feature(value):\n",
    "    return tf.train.Feature(float_list=tf.train.FloatList(value=value))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.        , 0.        , 0.        , 1.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 1.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 1.        , 1.        ,\n",
       "       0.05732417, 0.00669285, 0.9976241 , 0.96832377, 0.00682713,\n",
       "       0.14185107, 0.09885607, 0.00919371, 0.80059224, 0.00784502,\n",
       "       0.01763634, 0.55971366, 0.0541667 , 0.4378235 , 0.12786157,\n",
       "       0.16798161, 0.14931294, 0.01224884, 0.00532502, 0.26894143,\n",
       "       0.01551975, 0.        ])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load the data; I've saved it in my HOME/data/cpdb directory\n",
    "from pathlib import Path\n",
    "HOME = str(Path.home())\n",
    "\n",
    "# Reshape the data so we have (num_samples x max_seq_length x num_features)\n",
    "data = np.load(HOME+\"/data/cpdb/cpdb+profile_6133_filtered.npy.gz\").reshape(-1, 700, 57)\n",
    "num_samples = data.shape[0]\n",
    "\n",
    "# Display the second residue of the first sample\n",
    "data[0, 1, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.        , 0.        , 0.        , 1.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.05732417, 0.00669285, 0.9976241 ,\n",
       "       0.96832377, 0.00682713, 0.14185107, 0.09885607, 0.00919371,\n",
       "       0.80059224, 0.00784502, 0.01763634, 0.55971366, 0.0541667 ,\n",
       "       0.4378235 , 0.12786157, 0.16798161, 0.14931294, 0.01224884,\n",
       "       0.00532502, 0.26894143, 0.01551975])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# We're only interested in the amino acid and PSI-BLAST profile matrix features\n",
    "seqs = np.concatenate([data[:, :, 0:22].copy(), data[:, :, 35:56].copy()], axis=2).reshape(num_samples, -1)\n",
    "\n",
    "# There are 43 features per residue\n",
    "num_features = 43\n",
    "\n",
    "seqs[0, 43:86]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1., 0., 0., 0., 0., 0., 0., 0., 0.])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Reshape labels but don't flatten so we can count NoSeqs later\n",
    "labels = data[:, :, 22:31].copy().reshape(num_samples, 700, -1)\n",
    "labels[0, 1, :]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save as a TF Record\n",
    "We want to save each sample as a TF record. To save space and reduce training time, the\n",
    "padding from the original dataset (each protein was padded to 700 with 'NoSeq' tokens) \n",
    "is removed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Count the number of non-NoSeq tokens to find protein length\n",
    "noseq = np.array([[0., 0., 0., 0., 0., 0., 0., 0., 1.]])\n",
    "\n",
    "# Find all positions in the vector that are the 'NoSeq' label\n",
    "noseqs = np.all(np.equal(labels[0, :, :], noseq), axis=1)\n",
    "\n",
    "# Now negate the boolean values and count the number of Trues\n",
    "seq_length = np.logical_not(noseqs).sum()\n",
    "print(seq_length)\n",
    "\n",
    "noseqs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now that we know the length, we can flatten the labels\n",
    "labels = labels.reshape(num_samples, -1)\n",
    "\n",
    "# There are 9 labels\n",
    "num_labels = 9\n",
    "\n",
    "# Convert into a tf.train.Example, which would eventually be written to a file\n",
    "example = tf.train.Example(features=tf.train.Features(feature={\n",
    "    'seq_len': _int64_feature(seq_length),\n",
    "    'seq_data': _floats_feature(seqs[0, 0:num_features*seq_length]),\n",
    "    'label_data': _floats_feature(labels[0, 0:num_labels*seq_length])\n",
    "    }\n",
    "))\n",
    "example"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}