neighbor-knn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jehunyoo/AI/kakao/MelonPlaylistContinuation/main/neighbor_knn.py:7: UserWarning: Unsupported module 'tqdm' is used.\n",
      "  warn(\"Unsupported module 'tqdm' is used.\")\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from neighbor_knn import NeighborKNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "### 1. load data\n",
    "train = pd.read_json(\"res/train.json\")\n",
    "val = pd.read_json(\"res/val.json\")\n",
    "pred_songs = pd.read_json(\"submission/neighbor2_a7b0/neighbor2_a7b0_first.json\", orient='records')\n",
    "pred_tags = pd.read_json(\"submission/neighbor-tag_beta1_a7b0/neighbortag_beta1_a7b0.json\", orient='records')\n",
    "# test = pd.read_json(\"res/test.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9661\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tags</th>\n",
       "      <th>id</th>\n",
       "      <th>plylst_title</th>\n",
       "      <th>songs</th>\n",
       "      <th>like_cnt</th>\n",
       "      <th>updt_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[]</td>\n",
       "      <td>118598</td>\n",
       "      <td></td>\n",
       "      <td>[373313, 151080, 275346, 696876, 165237, 52593...</td>\n",
       "      <td>1675</td>\n",
       "      <td>2019-05-27 14:14:33.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[]</td>\n",
       "      <td>51464</td>\n",
       "      <td></td>\n",
       "      <td>[529437, 516103, 360067, 705713, 226062, 37089...</td>\n",
       "      <td>62</td>\n",
       "      <td>2008-06-21 23:26:22.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[]</td>\n",
       "      <td>45144</td>\n",
       "      <td></td>\n",
       "      <td>[589668, 21711, 570151, 320043, 13930, 599327,...</td>\n",
       "      <td>20</td>\n",
       "      <td>2017-10-30 18:15:43.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[]</td>\n",
       "      <td>79929</td>\n",
       "      <td></td>\n",
       "      <td>[672718, 121924, 102694, 683657, 201558, 38511...</td>\n",
       "      <td>20</td>\n",
       "      <td>2017-02-07 11:40:42.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>[]</td>\n",
       "      <td>127575</td>\n",
       "      <td></td>\n",
       "      <td>[227044, 424672, 515592, 22408, 57893, 153714,...</td>\n",
       "      <td>2</td>\n",
       "      <td>2011-10-28 11:21:51.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23003</th>\n",
       "      <td>[]</td>\n",
       "      <td>127349</td>\n",
       "      <td></td>\n",
       "      <td>[474446, 111476, 241237, 70500, 577313, 145058...</td>\n",
       "      <td>14</td>\n",
       "      <td>2018-02-02 17:43:37.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23006</th>\n",
       "      <td>[]</td>\n",
       "      <td>92067</td>\n",
       "      <td></td>\n",
       "      <td>[486938, 154124, 70969, 596414, 243733, 557848...</td>\n",
       "      <td>9</td>\n",
       "      <td>2013-02-16 01:07:41.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23012</th>\n",
       "      <td>[]</td>\n",
       "      <td>77438</td>\n",
       "      <td></td>\n",
       "      <td>[625875, 464051, 11657, 236393, 358186, 213435...</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-03-27 15:27:40.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23013</th>\n",
       "      <td>[]</td>\n",
       "      <td>36231</td>\n",
       "      <td></td>\n",
       "      <td>[161094, 665833, 688145, 432735, 439938, 12665...</td>\n",
       "      <td>31</td>\n",
       "      <td>2015-11-18 11:49:09.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23014</th>\n",
       "      <td>[]</td>\n",
       "      <td>65189</td>\n",
       "      <td></td>\n",
       "      <td>[643070, 132994, 98223, 293236, 513129, 650494...</td>\n",
       "      <td>19</td>\n",
       "      <td>2017-04-23 16:50:58.000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9661 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      tags      id plylst_title  \\\n",
       "0       []  118598                \n",
       "2       []   51464                \n",
       "3       []   45144                \n",
       "4       []   79929                \n",
       "6       []  127575                \n",
       "...    ...     ...          ...   \n",
       "23003   []  127349                \n",
       "23006   []   92067                \n",
       "23012   []   77438                \n",
       "23013   []   36231                \n",
       "23014   []   65189                \n",
       "\n",
       "                                                   songs  like_cnt  \\\n",
       "0      [373313, 151080, 275346, 696876, 165237, 52593...      1675   \n",
       "2      [529437, 516103, 360067, 705713, 226062, 37089...        62   \n",
       "3      [589668, 21711, 570151, 320043, 13930, 599327,...        20   \n",
       "4      [672718, 121924, 102694, 683657, 201558, 38511...        20   \n",
       "6      [227044, 424672, 515592, 22408, 57893, 153714,...         2   \n",
       "...                                                  ...       ...   \n",
       "23003  [474446, 111476, 241237, 70500, 577313, 145058...        14   \n",
       "23006  [486938, 154124, 70969, 596414, 243733, 557848...         9   \n",
       "23012  [625875, 464051, 11657, 236393, 358186, 213435...         0   \n",
       "23013  [161094, 665833, 688145, 432735, 439938, 12665...        31   \n",
       "23014  [643070, 132994, 98223, 293236, 513129, 650494...        19   \n",
       "\n",
       "                     updt_date  \n",
       "0      2019-05-27 14:14:33.000  \n",
       "2      2008-06-21 23:26:22.000  \n",
       "3      2017-10-30 18:15:43.000  \n",
       "4      2017-02-07 11:40:42.000  \n",
       "6      2011-10-28 11:21:51.000  \n",
       "...                        ...  \n",
       "23003  2018-02-02 17:43:37.000  \n",
       "23006  2013-02-16 01:07:41.000  \n",
       "23012  2019-03-27 15:27:40.000  \n",
       "23013  2015-11-18 11:49:09.000  \n",
       "23014  2017-04-23 16:50:58.000  \n",
       "\n",
       "[9661 rows x 6 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "9661"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index = []\n",
    "for i in val.index:\n",
    "    if val.songs[i] != [] and val.tags[i] == []:\n",
    "        index.append(i)\n",
    "print(len(index))\n",
    "new_val = val.iloc[index, :]\n",
    "display(new_val)\n",
    "new_val.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "### 2. modeling\n",
    "### 2.1 hyperparameters: k, rho, alpha, beta\n",
    "### 2.2 parameters: sim_songs, sim_tags, sim_normalize\n",
    "k = 100\n",
    "rho = 0.4\n",
    "alpha = 0.5\n",
    "beta = 0.5\n",
    "sim_songs = \"idf\"\n",
    "sim_tags = \"cos\"\n",
    "sim_normalize = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KNN version: NeighborKNN-1.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 1/9661 [00:03<10:05:07,  3.76s/it]\n"
     ]
    },
    {
     "ename": "KeyError",
     "evalue": "1",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-21-d71d9b5fadbb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      6\u001b[0m            \u001b[0msim_songs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msim_songs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msim_tags\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msim_tags\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msim_normalize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msim_normalize\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m            \u001b[0mtrain\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnew_val\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpred_songs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpred_songs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpred_tags\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpred_tags\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m            verbose=True, version_check=True).predict(start=0, end=new_val.shape[0], auto_save=False)\n\u001b[0m\u001b[1;32m      9\u001b[0m \u001b[0mdisplay\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpred\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/AI/kakao/MelonPlaylistContinuation/main/neighbor_knn.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, start, end, auto_save, auto_save_step, auto_save_fname)\u001b[0m\n\u001b[1;32m     90\u001b[0m             \u001b[0mk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 92\u001b[0;31m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mval_songs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0muth\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mval_tags\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0muth\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     93\u001b[0m                 pred.append({\n\u001b[1;32m     94\u001b[0m                 \u001b[0;34m\"id\"\u001b[0m \u001b[0;34m:\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mval_id\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0muth\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/cosmos/lib/python3.7/site-packages/pandas/core/series.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m    869\u001b[0m         \u001b[0mkey\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_if_callable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    870\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 871\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_value\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    872\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    873\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_scalar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/cosmos/lib/python3.7/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_value\u001b[0;34m(self, series, key)\u001b[0m\n\u001b[1;32m   4403\u001b[0m         \u001b[0mk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_convert_scalar_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkind\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"getitem\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4404\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 4405\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_value\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtz\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseries\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"tz\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   4406\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4407\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mholds_integer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_boolean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_value\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_value\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.Int64HashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.Int64HashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 1"
     ]
    }
   ],
   "source": [
    "### 3. range setting - KNN.predict()\n",
    "### 3.1 range(start, end); if end == None, then range(start, end of val)\n",
    "### 3.2 auto_save: boolean; False(default)\n",
    "### 3.3 return type of KNN.predict() : pandas.DataFrame\n",
    "pred = NeighborKNN(k=k, rho=rho, alpha=alpha, beta=beta, \\\n",
    "           sim_songs=sim_songs, sim_tags=sim_tags, sim_normalize=sim_normalize, \\\n",
    "           train=train, val=new_val, pred_songs=pred_songs, pred_tags=pred_tags, \\\n",
    "           verbose=True, version_check=True).predict(start=0, end=None, auto_save=False)\n",
    "display(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### 4. save data\n",
    "version = NeighborKNN.__version__\n",
    "version = version[version.find('-') + 1: version.find('.')]\n",
    "path = \"submission/neighbor-knn_k100rho4_idfcosFalse\"\n",
    "fname = f\"neighbor-knn{version}_k{k}rho{int(rho * 10)}_{sim_songs}{sim_tags}{sim_normalize}\"\n",
    "pred.to_json(f'{path}/{fname}.json', orient='records')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Only Tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from neighbor_knn_tag import NeighborKNN_tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "### 1. load data\n",
    "train = pd.read_json(\"res/train.json\")\n",
    "val = pd.read_json(\"res/val.json\")\n",
    "pred_songs = pd.read_json(\"submission/neighbor2_a7b0/neighbor2_a7b0_first.json\", orient='records')\n",
    "pred_tags = pd.read_json(\"submission/neighbor-tag_beta1_a7b1/neighbortag_beta1_a7b1.json\", orient='records')\n",
    "# test = pd.read_json(\"res/test.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tags</th>\n",
       "      <th>id</th>\n",
       "      <th>plylst_title</th>\n",
       "      <th>songs</th>\n",
       "      <th>like_cnt</th>\n",
       "      <th>updt_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[]</td>\n",
       "      <td>118598</td>\n",
       "      <td></td>\n",
       "      <td>[373313, 151080, 275346, 696876, 165237, 52593...</td>\n",
       "      <td>1675</td>\n",
       "      <td>2019-05-27 14:14:33.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[]</td>\n",
       "      <td>51464</td>\n",
       "      <td></td>\n",
       "      <td>[529437, 516103, 360067, 705713, 226062, 37089...</td>\n",
       "      <td>62</td>\n",
       "      <td>2008-06-21 23:26:22.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[]</td>\n",
       "      <td>45144</td>\n",
       "      <td></td>\n",
       "      <td>[589668, 21711, 570151, 320043, 13930, 599327,...</td>\n",
       "      <td>20</td>\n",
       "      <td>2017-10-30 18:15:43.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[]</td>\n",
       "      <td>79929</td>\n",
       "      <td></td>\n",
       "      <td>[672718, 121924, 102694, 683657, 201558, 38511...</td>\n",
       "      <td>20</td>\n",
       "      <td>2017-02-07 11:40:42.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>[]</td>\n",
       "      <td>127575</td>\n",
       "      <td></td>\n",
       "      <td>[227044, 424672, 515592, 22408, 57893, 153714,...</td>\n",
       "      <td>2</td>\n",
       "      <td>2011-10-28 11:21:51.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23003</th>\n",
       "      <td>[]</td>\n",
       "      <td>127349</td>\n",
       "      <td></td>\n",
       "      <td>[474446, 111476, 241237, 70500, 577313, 145058...</td>\n",
       "      <td>14</td>\n",
       "      <td>2018-02-02 17:43:37.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23006</th>\n",
       "      <td>[]</td>\n",
       "      <td>92067</td>\n",
       "      <td></td>\n",
       "      <td>[486938, 154124, 70969, 596414, 243733, 557848...</td>\n",
       "      <td>9</td>\n",
       "      <td>2013-02-16 01:07:41.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23012</th>\n",
       "      <td>[]</td>\n",
       "      <td>77438</td>\n",
       "      <td></td>\n",
       "      <td>[625875, 464051, 11657, 236393, 358186, 213435...</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-03-27 15:27:40.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23013</th>\n",
       "      <td>[]</td>\n",
       "      <td>36231</td>\n",
       "      <td></td>\n",
       "      <td>[161094, 665833, 688145, 432735, 439938, 12665...</td>\n",
       "      <td>31</td>\n",
       "      <td>2015-11-18 11:49:09.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23014</th>\n",
       "      <td>[]</td>\n",
       "      <td>65189</td>\n",
       "      <td></td>\n",
       "      <td>[643070, 132994, 98223, 293236, 513129, 650494...</td>\n",
       "      <td>19</td>\n",
       "      <td>2017-04-23 16:50:58.000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9661 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      tags      id  ... like_cnt                updt_date\n",
       "0       []  118598  ...     1675  2019-05-27 14:14:33.000\n",
       "2       []   51464  ...       62  2008-06-21 23:26:22.000\n",
       "3       []   45144  ...       20  2017-10-30 18:15:43.000\n",
       "4       []   79929  ...       20  2017-02-07 11:40:42.000\n",
       "6       []  127575  ...        2  2011-10-28 11:21:51.000\n",
       "...    ...     ...  ...      ...                      ...\n",
       "23003   []  127349  ...       14  2018-02-02 17:43:37.000\n",
       "23006   []   92067  ...        9  2013-02-16 01:07:41.000\n",
       "23012   []   77438  ...        0  2019-03-27 15:27:40.000\n",
       "23013   []   36231  ...       31  2015-11-18 11:49:09.000\n",
       "23014   []   65189  ...       19  2017-04-23 16:50:58.000\n",
       "\n",
       "[9661 rows x 6 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index = []\n",
    "for i in val.index:\n",
    "    if val.songs[i] != [] and val.tags[i] == []:\n",
    "        index.append(i)\n",
    "new_val = val.iloc[index, :].copy()\n",
    "new_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "### 2. modeling\n",
    "### 2.1 hyperparameters: k, rho, alpha, beta\n",
    "### 2.2 parameters: sim_songs, sim_tags, sim_normalize\n",
    "k = 100\n",
    "rho = 0.4\n",
    "sim_songs = \"idf\"\n",
    "sim_tags = \"cos\"\n",
    "sim_normalize = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KNN version: NeighborKNN-1.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6f33e10160654bf285e54d2f61d5e027",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=23015.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-17-c6acb016dbe6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      6\u001b[0m            \u001b[0msim_songs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msim_songs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msim_normalize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msim_normalize\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m            \u001b[0mtrain\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpred_songs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpred_songs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m            verbose=True, version_check=True).predict(start=0, end=None, auto_save=False)\n\u001b[0m\u001b[1;32m      9\u001b[0m \u001b[0mdisplay\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpred\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-16-81657fb8e149>\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, start, end, auto_save, auto_save_step, auto_save_fname)\u001b[0m\n\u001b[1;32m     91\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mval_songs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0muth\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mval_tags\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0muth\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m                 \u001b[0mplaylist_songs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpred_songs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0muth\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m                 \u001b[0msimSongs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mplaylist_songs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvplaylist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msim_songs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopt\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"songs\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mvplaylist\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mall_songs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     94\u001b[0m                 \u001b[0mtags\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-16-81657fb8e149>\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     91\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mval_songs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0muth\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mval_tags\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0muth\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m                 \u001b[0mplaylist_songs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpred_songs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0muth\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m                 \u001b[0msimSongs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mplaylist_songs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvplaylist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msim_songs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopt\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"songs\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mvplaylist\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mall_songs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     94\u001b[0m                 \u001b[0mtags\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-16-81657fb8e149>\u001b[0m in \u001b[0;36m_sim\u001b[0;34m(self, u, v, sim, opt)\u001b[0m\n\u001b[1;32m    142\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0mopt\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"tags\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    143\u001b[0m                 \u001b[0mfreq\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfreq_tags\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 144\u001b[0;31m             \u001b[0mfreq\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfreq\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mu\u001b[0m \u001b[0;34m&\u001b[0m \u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    145\u001b[0m             \u001b[0mfreq\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfreq\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m**\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrho\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# numpy!\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    146\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msim_normalize\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "### 3. range setting - KNN.predict()\n",
    "### 3.1 range(start, end); if end == None, then range(start, end of val)\n",
    "### 3.2 auto_save: boolean; False(default)\n",
    "### 3.3 return type of KNN.predict() : pandas.DataFrame\n",
    "pred = NeighborKNN(k=k, rho=rho, \\\n",
    "           sim_songs=sim_songs, sim_normalize=sim_normalize, \\\n",
    "           train=train, val=val, pred_songs=pred_songs, \\\n",
    "           verbose=True, version_check=True).predict(start=0, end=None, auto_save=False)\n",
    "display(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jehunyoo/anaconda3/envs/cosmos/lib/python3.7/site-packages/ipykernel_launcher.py:7: UserWarning: Unsupported module 'tqdm' is used.\n",
      "  import sys\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "from collections import Counter\n",
    "from warnings import warn\n",
    "\n",
    "warn(\"Unsupported module 'tqdm' is used.\")\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "\n",
    "class NeighborKNN:\n",
    "\n",
    "    __version__ = \"NeighborKNN-1.0\"\n",
    "    \n",
    "    def __init__(self, k, rho=0.4, \\\n",
    "                 sim_songs=\"idf\", sim_normalize=False, \\\n",
    "                 train=None, val=None, \\\n",
    "                 pred_songs=None, \\\n",
    "                 verbose=True, version_check=True):\n",
    "        '''\n",
    "        k : int\n",
    "        rho : float; 0.4(default) only for idf\n",
    "        alpha, beta : float; 0.5(default)\n",
    "        sim_songs, sim_tags : \"cos\"(default), \"idf\", \"jaccard\"\n",
    "        sim_normalize : boolean; when sim == \"cos\" or \"idf\"\n",
    "        verbose : boolean\n",
    "        '''\n",
    "        self.train_id = train[\"id\"].copy()\n",
    "        self.train_songs = train[\"songs\"].copy()\n",
    "        self.train_tags = train[\"tags\"].copy()\n",
    "        del train\n",
    "\n",
    "        self.val_id = val[\"id\"].copy()\n",
    "        self.val_songs = val[\"songs\"].copy()\n",
    "        self.val_tags = val[\"tags\"].copy()\n",
    "        del val\n",
    "        \n",
    "        self.pred_songs = pred_songs[\"tags\"].copy()\n",
    "\n",
    "        self.freq_songs = None\n",
    "        self.freq_tags = None\n",
    "        \n",
    "        self.k = k\n",
    "        self.rho = rho\n",
    "        self.alpha = alpha\n",
    "        self.beta = beta\n",
    "\n",
    "        self.sim_songs = sim_songs\n",
    "        self.sim_normalize = sim_normalize\n",
    "\n",
    "        self.verbose = verbose\n",
    "        self.__version__ = NeighborKNN.__version__\n",
    "\n",
    "        if version_check:\n",
    "            print(f\"KNN version: {NeighborKNN.__version__}\")\n",
    "\n",
    "        TOTAL_SONGS = 707989      # total number of songs\n",
    "\n",
    "        if self.sim_songs == \"idf\":\n",
    "\n",
    "            self.freq_songs = np.zeros(TOTAL_SONGS, dtype=np.int64)\n",
    "            _playlist = self.train_songs\n",
    "            for _songs in _playlist:\n",
    "                self.freq_songs[_songs] += 1\n",
    "\n",
    "\n",
    "    def predict(self, start=0, end=None, auto_save=False, auto_save_step=500, auto_save_fname='auto_save'):\n",
    "        '''\n",
    "        start, end : range(start, end). if end = None, range(start, end of val)\n",
    "        auto_save : boolean; False(default)\n",
    "        auto_save_step : int; 500(default)\n",
    "        auto_save_fname : string (without extension); 'auto_save'(default)\n",
    "        @returns : pandas.DataFrame; columns=['id', 'songs', 'tags']\n",
    "        '''\n",
    "\n",
    "        # TODO: Remove unsupported module 'tqdm'.\n",
    "        if end:\n",
    "            _range = tqdm(range(start, end)) if self.verbose else range(start, end)\n",
    "        elif end == None:\n",
    "            _range = tqdm(range(start, self.val_id.index.stop)) if self.verbose else range(start, self.val_id.index.stop)\n",
    "\n",
    "        pred = []\n",
    "        all_songs = [set(songs) for songs in self.train_songs] # list of set\n",
    "        all_tags =  [set(tags) for tags in self.train_tags]    # list of set\n",
    "\n",
    "        for uth in _range:\n",
    "\n",
    "            k = self.k\n",
    "            \n",
    "            # predict tags using songs\n",
    "            if self.val_songs[uth] != [] and self.val_tags[uth] == []:\n",
    "                playlist_songs = set(self.pred_songs[uth])\n",
    "                simSongs = np.array([self._sim(playlist_songs, vplaylist, self.sim_songs, opt=\"songs\") for vplaylist in all_songs])\n",
    "                tags = []\n",
    "                \n",
    "                while len(tags) < 10:\n",
    "                    top = simSongs.argsort()[-k:]\n",
    "                    _tags = []\n",
    "                    \n",
    "                    for vth in top:\n",
    "                        _tags += self.train_tags[vth]\n",
    "\n",
    "                    counts = Counter(_tags).most_common(30)\n",
    "                    tags = [tag for tag, _ in counts]\n",
    "\n",
    "                    k += 100\n",
    "                \n",
    "                pred_tags = tags[:10]\n",
    "\n",
    "                pred.append({\n",
    "                \"id\" : int(self.val_id[uth]),\n",
    "                \"songs\" : self.pred_songs[uth],\n",
    "                \"tags\" : pred_tags\n",
    "                })\n",
    "\n",
    "            if (auto_save == True) and ((uth + 1) % auto_save_step == 0):\n",
    "                self._auto_save(pred, auto_save_fname)\n",
    "        \n",
    "        return pd.DataFrame(pred)\n",
    "    \n",
    "\n",
    "    def _sim(self, u, v, sim, opt):\n",
    "        '''\n",
    "        u : set (playlist in train data)\n",
    "        v : set (playlist in test data)\n",
    "        sim : string; \"cos\", \"idf\", \"jaccard\" (kind of similarity)\n",
    "        opt : string; \"songs\", \"tags\"\n",
    "        '''\n",
    "\n",
    "        if sim == \"cos\":\n",
    "            if self.sim_normalize:\n",
    "                try:\n",
    "                    len(u & v) / ((len(u) ** 0.5) * (len(v) ** 0.5))\n",
    "                except:\n",
    "                    return 0\n",
    "            else:\n",
    "                return len(u & v)\n",
    "        \n",
    "        elif sim == \"idf\":\n",
    "            if opt == \"songs\":\n",
    "                freq = self.freq_songs\n",
    "            elif opt == \"tags\":\n",
    "                freq = self.freq_tags\n",
    "            freq = freq[list(u & v)]\n",
    "            freq = 1 / (((freq - 1) ** self.rho) + 1) # numpy!\n",
    "            if self.sim_normalize:\n",
    "                try:\n",
    "                    return freq.sum() / ((len(u) ** 0.5) * (len(v) ** 0.5))\n",
    "                except:\n",
    "                    return 0\n",
    "            else:\n",
    "                return freq.sum()\n",
    "        \n",
    "        elif sim == \"jaccard\":\n",
    "            return len(u & v) / len(u | v)\n",
    "    \n",
    "    def _auto_save(self, pred, auto_save_fname):\n",
    "        '''\n",
    "        pred : list of dictionaries\n",
    "        auto_save_fname : string\n",
    "        '''\n",
    "        \n",
    "        if not os.path.isdir(\"./_temp\"):\n",
    "            os.mkdir('./_temp')\n",
    "        pd.DataFrame(pred).to_json(f'_temp/{auto_save_fname}.json', orient='records')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}