scalability-test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gc\n",
    "import time\n",
    "import pickle\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.manifold import TSNE\n",
    "from sklearn.cluster import DBSCAN\n",
    "from sklearn import svm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "time_checks = [10, 50, 100, 150, 200, 250, 300, 350, 400, 450, 500]#, 500000, 1000000] # number of samples to be generated\n",
    "sizes = [1, 2, 3, 4, 5, 6, 7, 8, 31]\n",
    "\n",
    "# best OCSVM model\n",
    "kernel = 'linear'\n",
    "nu = .01\n",
    "gamma = .5\n",
    "clf_ocsvm = {}\n",
    "for n_components in sizes:\n",
    "    data = np.random.normal(size=(time_checks[0], n_components))\n",
    "    clf_ocsvm[n_components] = svm.OneClassSVM(nu=nu, kernel=kernel, gamma=gamma)\n",
    "    clf_ocsvm[n_components].fit(data)\n",
    "\n",
    "# best DBSCAN model\n",
    "min_samples = 5\n",
    "epsilon = 1.\n",
    "clf_dbscan = DBSCAN(eps=epsilon, min_samples=min_samples, metric='euclidean')\n",
    "\n",
    "# data from encoder\n",
    "layer = 400, 100, 40\n",
    "dataset_filter = 'normal'\n",
    "optimizer_name = 'SGD'\n",
    "lr = 0.01\n",
    "base_autoencoder_model = r'./models/{}_{}_{}_encoder_{}-model.h5'\n",
    "\n",
    "dimensionality_reduction = ['pca', 'tsne', 'ae']\n",
    "algorithm = ['ocsvm', 'dbscan']\n",
    "scalability_results = {}\n",
    "scalability_results_full = {}\n",
    "for alg in algorithm:\n",
    "    scalability_results[alg] = {}\n",
    "    scalability_results_full[alg] = {}\n",
    "    for dim in dimensionality_reduction:\n",
    "        scalability_results[alg][dim] = {}\n",
    "        for n_components in sizes:\n",
    "            if n_components == 31: continue\n",
    "            scalability_results[alg][dim][n_components] = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gc: 55185\n",
      "done 10\n",
      "gc: 55128\n",
      "done 50\n",
      "gc: 82704\n",
      "done 100\n",
      "gc: 68904\n",
      "done 150\n",
      "gc: 55104\n",
      "done 200\n",
      "gc: 55113\n",
      "done 250\n",
      "gc: 41328\n",
      "done 300\n",
      "gc: 27561\n",
      "done 350\n",
      "gc: 27561\n",
      "done 400\n",
      "gc: 27552\n",
      "done 450\n"
     ]
    },
    {
     "ename": "MemoryError",
     "evalue": "Unable to allocate array with shape (1249975000,) and data type float64",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-6fe3f568376b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     53\u001b[0m         \u001b[0mstart_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime_ns\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m         \u001b[0mtsne\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTSNE\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_components\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mn_components\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_iter\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m300\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'exact'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 55\u001b[0;31m         \u001b[0mtsne_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtsne\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     56\u001b[0m         \u001b[0mtime_tsne\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime_ns\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mstart_time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/venv37-tf20-cpu/lib/python3.7/site-packages/sklearn/manifold/_t_sne.py\u001b[0m in \u001b[0;36mfit_transform\u001b[0;34m(self, X, y)\u001b[0m\n\u001b[1;32m    884\u001b[0m             \u001b[0mEmbedding\u001b[0m \u001b[0mof\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mtraining\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mlow\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mdimensional\u001b[0m \u001b[0mspace\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    885\u001b[0m         \"\"\"\n\u001b[0;32m--> 886\u001b[0;31m         \u001b[0membedding\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    887\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0membedding_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0membedding\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    888\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0membedding_\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/venv37-tf20-cpu/lib/python3.7/site-packages/sklearn/manifold/_t_sne.py\u001b[0m in \u001b[0;36m_fit\u001b[0;34m(self, X, skip_num_points)\u001b[0m\n\u001b[1;32m    721\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    722\u001b[0m             \u001b[0;31m# compute the joint probability distribution for the input space\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 723\u001b[0;31m             \u001b[0mP\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_joint_probabilities\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdistances\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mperplexity\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    724\u001b[0m             \u001b[0;32massert\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misfinite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mP\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"All probabilities should be finite\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    725\u001b[0m             \u001b[0;32massert\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mP\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"All probabilities should be non-negative\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/venv37-tf20-cpu/lib/python3.7/site-packages/sklearn/manifold/_t_sne.py\u001b[0m in \u001b[0;36m_joint_probabilities\u001b[0;34m(distances, desired_perplexity, verbose)\u001b[0m\n\u001b[1;32m     58\u001b[0m     \u001b[0mP\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconditional_P\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mconditional_P\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mT\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m     \u001b[0msum_P\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmaximum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mP\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mMACHINE_EPSILON\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 60\u001b[0;31m     \u001b[0mP\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmaximum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msquareform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mP\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0msum_P\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mMACHINE_EPSILON\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     61\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mP\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mMemoryError\u001b[0m: Unable to allocate array with shape (1249975000,) and data type float64"
     ]
    }
   ],
   "source": [
    "for num_samples in time_checks:\n",
    "    data = np.random.normal(size=(num_samples*100, 31))\n",
    "    \n",
    "    start_time = time.time_ns()\n",
    "    y = clf_ocsvm[31].predict(data[:num_samples])\n",
    "    scalability_results_full['ocsvm'][num_samples] = time.time_ns() - start_time\n",
    "    \n",
    "    clf = DBSCAN(eps=epsilon, min_samples=min_samples, metric='euclidean')\n",
    "    start_time = time.time_ns()\n",
    "    y = clf.fit_predict(data)\n",
    "    scalability_results_full['dbscan'][num_samples] = time.time_ns() - start_time\n",
    "    \n",
    "    for n_components in sizes:\n",
    "        if n_components == 31: continue\n",
    "        \n",
    "        pca = PCA(n_components=n_components)\n",
    "        start_time = time.time_ns()\n",
    "        pca_data = pca.fit_transform(data[:num_samples])\n",
    "        time_pca = time.time_ns() - start_time\n",
    "        \n",
    "        tsne = TSNE(n_components=n_components, verbose=0, n_iter=300, method='exact')\n",
    "        start_time = time.time_ns()\n",
    "        tsne_data = tsne.fit_transform(data[:num_samples])\n",
    "        time_tsne = time.time_ns() - start_time\n",
    "        \n",
    "        layers = layer + (n_components,)\n",
    "        model = tf.keras.models.load_model(base_autoencoder_model.format(dataset_filter, optimizer_name, lr, '-'.join(str(x) for x in layers)))\n",
    "        output_layer = len(model.layers) // 2\n",
    "        encoder = tf.keras.Model(inputs=model.input, outputs=model.layers[output_layer-1].output)\n",
    "#         encoder.summary()\n",
    "        start_time = time.time_ns()\n",
    "        encoded_data = encoder.predict(data[:num_samples])\n",
    "        time_encoder = time.time_ns() - start_time\n",
    "        \n",
    "        start_time = time.time_ns()\n",
    "        y_pred = clf_ocsvm[n_components].predict(pca_data)\n",
    "        scalability_results['ocsvm']['pca'][n_components][num_samples] = time.time_ns() - start_time + time_pca\n",
    "        \n",
    "        start_time = time.time_ns()\n",
    "        y_pred = clf_ocsvm[n_components].predict(tsne_data)\n",
    "        scalability_results['ocsvm']['tsne'][n_components][num_samples] = time.time_ns() - start_time + time_tsne\n",
    "        \n",
    "        start_time = time.time_ns()\n",
    "        y_pred = clf_ocsvm[n_components].predict(encoded_data)\n",
    "        scalability_results['ocsvm']['ae'][n_components][num_samples] = time.time_ns() - start_time + time_encoder\n",
    "        \n",
    "        # DBSCAN\n",
    "        start_time = time.time_ns()\n",
    "        pca = PCA(n_components=n_components)\n",
    "        pca_data = pca.fit_transform(data)\n",
    "        time_pca = time.time_ns() - start_time\n",
    "        \n",
    "        start_time = time.time_ns()\n",
    "        tsne = TSNE(n_components=n_components, verbose=0, n_iter=300, method='exact')\n",
    "        tsne_data = tsne.fit_transform(data)\n",
    "        time_tsne = time.time_ns() - start_time\n",
    "        \n",
    "        layers = layer + (n_components,)\n",
    "        model = tf.keras.models.load_model(base_autoencoder_model.format(dataset_filter, optimizer_name, lr, '-'.join(str(x) for x in layers)))\n",
    "        output_layer = len(model.layers) // 2\n",
    "        encoder = tf.keras.Model(inputs=model.input, outputs=model.layers[output_layer-1].output)\n",
    "#         encoder.summary()\n",
    "        start_time = time.time_ns()\n",
    "        encoded_data = encoder.predict(data)\n",
    "        time_encoder = time.time_ns() - start_time\n",
    "        \n",
    "        clf = DBSCAN(eps=epsilon, min_samples=min_samples, metric='euclidean')\n",
    "        \n",
    "        start_time = time.time_ns()\n",
    "        y_pred = clf.fit_predict(pca_data)\n",
    "        scalability_results['dbscan']['pca'][n_components][num_samples] = time.time_ns() - start_time + time_pca\n",
    "        \n",
    "        start_time = time.time_ns()\n",
    "        y_pred = clf.fit_predict(tsne_data)\n",
    "        scalability_results['dbscan']['tsne'][n_components][num_samples] = time.time_ns() - start_time + time_tsne\n",
    "        \n",
    "        start_time = time.time_ns()\n",
    "        y_pred = clf.fit_predict(encoded_data)\n",
    "        scalability_results['dbscan']['ae'][n_components][num_samples] = time.time_ns() - start_time + time_encoder\n",
    "        \n",
    "    del data, pca_data, tsne_data, encoded_data\n",
    "    print('gc:', gc.collect())\n",
    "    \n",
    "    with open('./models/scalability_results.h5', 'wb') as file:\n",
    "        pickle.dump({'scalability_results': scalability_results,\n",
    "                    'scalability_results_full': scalability_results_full}, file)\n",
    "    print('done', num_samples)\n",
    "#     break"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7 TF 2 CPU",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}