dataset_download_GCP.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "vHD5xYC0MByx"
   },
   "source": [
    "Scripts for downloading online datesets into Google drive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "_5CaH5AkMdkk"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tarfile\n",
    "import urllib.request\n",
    "\n",
    "from six.moves import urllib\n",
    "local_data_dir = \"./Datasets/\" "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "CZImadqFQm8u",
    "outputId": "b78fac21-8c04-45de-bf24-61535d38073f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "extracted data files to ./Datasets\n"
     ]
    }
   ],
   "source": [
    "#CIFAR-10\n",
    "data_path = \"https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz\"\n",
    "url = urllib.request.urlopen(data_path)\n",
    "archive = tarfile.open(fileobj=url, mode='r|gz') # read a .tar.gz stream\n",
    "archive.extractall(local_data_dir)\n",
    "url.close()\n",
    "archive.close()\n",
    "print('extracted data files to %s' % local_data_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "extracted data files to ./Datasets/\n"
     ]
    }
   ],
   "source": [
    "#svhn dataset \n",
    "urllib.request.urlretrieve(\"http://ufldl.stanford.edu/housenumbers/train_32x32.mat\", local_data_dir+\"train_32x32.mat\")\n",
    "urllib.request.urlretrieve(\"http://ufldl.stanford.edu/housenumbers/test_32x32.mat\", local_data_dir+\"test_32x32.mat\")\n",
    "print('extracted data files to %s' % local_data_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# svhn reprocessin\n",
    "train_mat = sio.loadmat(local_data_dir + \"train_32x32.mat\")\n",
    "train_images_array = np.transpose(train_mat['X'], [3, 0, 1, 2])\n",
    "train_labels_array = train_mat[\"y\"]-1\n",
    "train_data_dict = {'images':train_images_array, 'labels':train_labels_array}\n",
    "\n",
    "test_mat = sio.loadmat(local_data_dir  + \"test_32x32.mat\")\n",
    "test_images_array = np.transpose(test_mat['X'], [3, 0, 1, 2])\n",
    "test_labels_array = test_mat[\"y\"]-1\n",
    "test_data_dict = {'images':test_images_array, 'labels':test_labels_array}\n",
    "\n",
    "np.save(local_data_dir+'train_data_dict.npy', train_data_dict)\n",
    "np.save(local_data_dir+'test_data_dict.npy', test_data_dict)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "dataset_downloading.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}