1. VIST_data preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-07T08:06:00.415077Z",
     "start_time": "2020-08-07T08:05:58.764491Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import numpy as np\n",
    "import nltk\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn import datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-07T08:06:02.955285Z",
     "start_time": "2020-08-07T08:06:00.999515Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "json_data len:  5\n",
      "json_data type:  <class 'dict'>\n",
      "dict_keys(['images', 'info', 'albums', 'type', 'annotations'])\n"
     ]
    }
   ],
   "source": [
    "# Json data\n",
    "\n",
    "with open('D:\\\\Visual_Storytelling_Challenge2018\\\\text\\\\stories\\\\train.story-in-sequence.json',\"r\") as data_file:\n",
    "    raw_data = data_file.readlines() # type list, len 1\n",
    "    # raw_data[0] type str, len 143752841\n",
    "    json_data = json.loads(raw_data[0])  # the JSON object must be str, not 'list'\n",
    "    print (\"json_data len: \", len(json_data))\n",
    "    print (\"json_data type: \", type(json_data))\n",
    "    print (json_data.keys())  # annotations = 연속되는 5 가지 사진에 대한 스토리 텍스트"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-07T08:06:02.965260Z",
     "start_time": "2020-08-07T08:06:02.956282Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len:  8031\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'date_create': '1214980972',\n",
       " 'date_update': '1432330952',\n",
       " 'description': 'Believed to be the largest Moreton Bay Fig Tree in the united States',\n",
       " 'farm': '4',\n",
       " 'id': '72157605930515606',\n",
       " 'owner': '12806074@N08',\n",
       " 'photos': '13',\n",
       " 'primary': '2626985925',\n",
       " 'secret': '98149cd59b',\n",
       " 'server': '3104',\n",
       " 'title': 'Moreton Bay Fig 1877',\n",
       " 'vist_label': '4th_of_july'}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "albums_list = json_data['albums']\n",
    "print (\"len: \", len(albums_list))\n",
    "albums_list[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-07T08:06:02.980218Z",
     "start_time": "2020-08-07T08:06:02.967254Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len:  167528\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'album_id': '72157605930515606',\n",
       " 'datetaken': '2008-06-30 07:33:43',\n",
       " 'id': '2626977325',\n",
       " 'latitude': '34.414760',\n",
       " 'license': '5',\n",
       " 'longitude': '-119.692879',\n",
       " 'media': 'photo',\n",
       " 'secret': 'bec0ff3596',\n",
       " 'tags': 'santabarbara',\n",
       " 'text': '',\n",
       " 'title': 'Moreton Bay Fig 1877',\n",
       " 'url_o': 'https://farm3.staticflickr.com/2078/2626977325_2b7696990c_o.jpg'}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image_list = json_data['images']\n",
    "print (\"len: \", len(image_list))\n",
    "image_list[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-07T08:06:02.993183Z",
     "start_time": "2020-08-07T08:06:02.981215Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len:  200775\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'album_id': '72157605930515606',\n",
       "  'original_text': 'Our landmark tree in town was about to be destroyed and cleared for a new mall. ',\n",
       "  'photo_flickr_id': '2627795780',\n",
       "  'setting': 'first-2-pick-and-tell',\n",
       "  'story_id': '30355',\n",
       "  'storylet_id': '151775',\n",
       "  'text': 'our landmark tree in town was about to be destroyed and cleared for a new mall .',\n",
       "  'tier': 'story-in-sequence',\n",
       "  'worker_arranged_photo_order': 0,\n",
       "  'worker_id': 'SY6QQXJCXXMNCYP'}]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 원하는 정보\n",
    "annotation_list = json_data['annotations']\n",
    "print (\"len: \", len(annotation_list))\n",
    "annotation_list[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-07T08:06:03.005152Z",
     "start_time": "2020-08-07T08:06:02.994180Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "type:  story-in-sequence\n",
      "info:  SIND v1.0\n"
     ]
    }
   ],
   "source": [
    "print(\"type: \", json_data['type'])\n",
    "print(\"info: \", json_data['info'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-07T08:06:04.462255Z",
     "start_time": "2020-08-07T08:06:03.673365Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "df_story_train:  (200775, 8)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>worker_id</th>\n",
       "      <th>album_id</th>\n",
       "      <th>story_id</th>\n",
       "      <th>storylet_id</th>\n",
       "      <th>photo_flickr_id</th>\n",
       "      <th>worker_arranged_photo_order</th>\n",
       "      <th>original_text</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SY6QQXJCXXMNCYP</td>\n",
       "      <td>72157605930515606</td>\n",
       "      <td>30355</td>\n",
       "      <td>151775</td>\n",
       "      <td>2627795780</td>\n",
       "      <td>0</td>\n",
       "      <td>Our landmark tree in town was about to be dest...</td>\n",
       "      <td>our landmark tree in town was about to be dest...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SY6QQXJCXXMNCYP</td>\n",
       "      <td>72157605930515606</td>\n",
       "      <td>30355</td>\n",
       "      <td>151776</td>\n",
       "      <td>2626979987</td>\n",
       "      <td>1</td>\n",
       "      <td>So we decided to take the day to go out and en...</td>\n",
       "      <td>so we decided to take the day to go out and en...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>SY6QQXJCXXMNCYP</td>\n",
       "      <td>72157605930515606</td>\n",
       "      <td>30355</td>\n",
       "      <td>151777</td>\n",
       "      <td>2626982337</td>\n",
       "      <td>2</td>\n",
       "      <td>To see the final glimpse of the roots, extendi...</td>\n",
       "      <td>to see the final glimpse of the roots , extend...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>SY6QQXJCXXMNCYP</td>\n",
       "      <td>72157605930515606</td>\n",
       "      <td>30355</td>\n",
       "      <td>151778</td>\n",
       "      <td>2626983575</td>\n",
       "      <td>3</td>\n",
       "      <td>And its magnificent trunk, larger than life it...</td>\n",
       "      <td>and its magnificent trunk , larger than life i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>SY6QQXJCXXMNCYP</td>\n",
       "      <td>72157605930515606</td>\n",
       "      <td>30355</td>\n",
       "      <td>151779</td>\n",
       "      <td>2626985925</td>\n",
       "      <td>4</td>\n",
       "      <td>One last picture of its beauty so we could cap...</td>\n",
       "      <td>one last picture of its beauty so we could cap...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         worker_id           album_id story_id storylet_id photo_flickr_id  \\\n",
       "0  SY6QQXJCXXMNCYP  72157605930515606    30355      151775      2627795780   \n",
       "1  SY6QQXJCXXMNCYP  72157605930515606    30355      151776      2626979987   \n",
       "2  SY6QQXJCXXMNCYP  72157605930515606    30355      151777      2626982337   \n",
       "3  SY6QQXJCXXMNCYP  72157605930515606    30355      151778      2626983575   \n",
       "4  SY6QQXJCXXMNCYP  72157605930515606    30355      151779      2626985925   \n",
       "\n",
       "   worker_arranged_photo_order  \\\n",
       "0                            0   \n",
       "1                            1   \n",
       "2                            2   \n",
       "3                            3   \n",
       "4                            4   \n",
       "\n",
       "                                       original_text  \\\n",
       "0  Our landmark tree in town was about to be dest...   \n",
       "1  So we decided to take the day to go out and en...   \n",
       "2  To see the final glimpse of the roots, extendi...   \n",
       "3  And its magnificent trunk, larger than life it...   \n",
       "4  One last picture of its beauty so we could cap...   \n",
       "\n",
       "                                                text  \n",
       "0  our landmark tree in town was about to be dest...  \n",
       "1  so we decided to take the day to go out and en...  \n",
       "2  to see the final glimpse of the roots , extend...  \n",
       "3  and its magnificent trunk , larger than life i...  \n",
       "4  one last picture of its beauty so we could cap...  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df_story_train 만들기\n",
    "column_list = ['worker_id', 'album_id', 'story_id', 'storylet_id', 'photo_flickr_id',\n",
    "               'worker_arranged_photo_order', 'original_text', 'text']\n",
    "total_row_list = []\n",
    "\n",
    "for i in range(len(annotation_list)):\n",
    "    row_list = []\n",
    "    for column in column_list:\n",
    "        row_list.append(annotation_list[i][0][column])\n",
    "    \n",
    "    total_row_list.append(row_list)\n",
    "    \n",
    "df_story_train = pd.DataFrame(total_row_list, columns = column_list)\n",
    "print(\"df_story_train: \", df_story_train.shape)\n",
    "df_story_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-07T08:06:04.581941Z",
     "start_time": "2020-08-07T08:06:04.463253Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "photo_flickr_id:  64934\n",
      "album_id:  8031\n",
      "story_id:  40155\n",
      "storylet_id:  200775\n"
     ]
    }
   ],
   "source": [
    "print(\"photo_flickr_id: \", len(df_story_train[\"photo_flickr_id\"].unique()))\n",
    "print(\"album_id: \", len(df_story_train[\"album_id\"].unique()))\n",
    "print(\"story_id: \", len(df_story_train[\"story_id\"].unique()))\n",
    "print(\"storylet_id: \", len(df_story_train[\"storylet_id\"].unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-07T08:06:05.431663Z",
     "start_time": "2020-08-07T08:06:05.298021Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "df_description_train:  (120465, 6)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>worker_id</th>\n",
       "      <th>album_id</th>\n",
       "      <th>photo_flickr_id</th>\n",
       "      <th>photo_order_in_story</th>\n",
       "      <th>original_text</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BNCNALGFUMO75H3</td>\n",
       "      <td>72157605930515606</td>\n",
       "      <td>2701863545</td>\n",
       "      <td>0</td>\n",
       "      <td>Large tree with many outstretching branches an...</td>\n",
       "      <td>large tree with many outstretching branches an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>LLGWHRGGCUXGHCQ</td>\n",
       "      <td>72157605930515606</td>\n",
       "      <td>2626977325</td>\n",
       "      <td>1</td>\n",
       "      <td>A green sign is describing a historic tree and...</td>\n",
       "      <td>a green sign is describing a historic tree and...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NMDHPHI897FIBCX</td>\n",
       "      <td>72157605930515606</td>\n",
       "      <td>2627795780</td>\n",
       "      <td>2</td>\n",
       "      <td>A large tree with roots that look like crocodi...</td>\n",
       "      <td>a large tree with roots that look like crocodi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>K4PTO2GKQ6BUJDA</td>\n",
       "      <td>72157605930515606</td>\n",
       "      <td>2626983575</td>\n",
       "      <td>3</td>\n",
       "      <td>Big old tree being photographed on a sunny day</td>\n",
       "      <td>big old tree being photographed on a sunny day</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4948BW6755GBL81</td>\n",
       "      <td>72157605930515606</td>\n",
       "      <td>2626982337</td>\n",
       "      <td>4</td>\n",
       "      <td>Huge brown tree roots rose above the ground.</td>\n",
       "      <td>huge brown tree roots rose above the ground .</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>N04A7HOMFBZR0MW</td>\n",
       "      <td>72157605930515606</td>\n",
       "      <td>2701863545</td>\n",
       "      <td>0</td>\n",
       "      <td>A large tree with many branches coming out</td>\n",
       "      <td>a large tree with many branches coming out</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>QKDKWV66QKOG7GH</td>\n",
       "      <td>72157605930515606</td>\n",
       "      <td>2626977325</td>\n",
       "      <td>1</td>\n",
       "      <td>A plaque describes an historical tree and advi...</td>\n",
       "      <td>a plaque describes an historical tree and advi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0LAO894GS3KUDK2</td>\n",
       "      <td>72157605930515606</td>\n",
       "      <td>2627795780</td>\n",
       "      <td>2</td>\n",
       "      <td>A tree has spread its roots high and wide acro...</td>\n",
       "      <td>a tree has spread its roots high and wide acro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>S8235PXB9Z3JZYL</td>\n",
       "      <td>72157605930515606</td>\n",
       "      <td>2626983575</td>\n",
       "      <td>3</td>\n",
       "      <td>A old curvy tree in the sun light .</td>\n",
       "      <td>a old curvy tree in the sun light .</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>N13ZJSXRVKEV0MD</td>\n",
       "      <td>72157605930515606</td>\n",
       "      <td>2626982337</td>\n",
       "      <td>4</td>\n",
       "      <td>Some trees that are surrounded by apartment bu...</td>\n",
       "      <td>some trees that are surrounded by apartment bu...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         worker_id           album_id photo_flickr_id photo_order_in_story  \\\n",
       "0  BNCNALGFUMO75H3  72157605930515606      2701863545                    0   \n",
       "1  LLGWHRGGCUXGHCQ  72157605930515606      2626977325                    1   \n",
       "2  NMDHPHI897FIBCX  72157605930515606      2627795780                    2   \n",
       "3  K4PTO2GKQ6BUJDA  72157605930515606      2626983575                    3   \n",
       "4  4948BW6755GBL81  72157605930515606      2626982337                    4   \n",
       "5  N04A7HOMFBZR0MW  72157605930515606      2701863545                    0   \n",
       "6  QKDKWV66QKOG7GH  72157605930515606      2626977325                    1   \n",
       "7  0LAO894GS3KUDK2  72157605930515606      2627795780                    2   \n",
       "8  S8235PXB9Z3JZYL  72157605930515606      2626983575                    3   \n",
       "9  N13ZJSXRVKEV0MD  72157605930515606      2626982337                    4   \n",
       "\n",
       "                                       original_text  \\\n",
       "0  Large tree with many outstretching branches an...   \n",
       "1  A green sign is describing a historic tree and...   \n",
       "2  A large tree with roots that look like crocodi...   \n",
       "3     Big old tree being photographed on a sunny day   \n",
       "4       Huge brown tree roots rose above the ground.   \n",
       "5         A large tree with many branches coming out   \n",
       "6  A plaque describes an historical tree and advi...   \n",
       "7  A tree has spread its roots high and wide acro...   \n",
       "8                A old curvy tree in the sun light .   \n",
       "9  Some trees that are surrounded by apartment bu...   \n",
       "\n",
       "                                                text  \n",
       "0  large tree with many outstretching branches an...  \n",
       "1  a green sign is describing a historic tree and...  \n",
       "2  a large tree with roots that look like crocodi...  \n",
       "3     big old tree being photographed on a sunny day  \n",
       "4      huge brown tree roots rose above the ground .  \n",
       "5         a large tree with many branches coming out  \n",
       "6  a plaque describes an historical tree and advi...  \n",
       "7  a tree has spread its roots high and wide acro...  \n",
       "8                a old curvy tree in the sun light .  \n",
       "9  some trees that are surrounded by apartment bu...  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# description data 불러오기\n",
    "import pickle\n",
    "\n",
    "with open('D:\\\\Visual_Storytelling_Challenge2018\\\\code\\\\test\\\\df_description.pickle', 'rb') as f:\n",
    "    df_description_train = pickle.load(f)\n",
    "\n",
    "print(\"df_description_train: \", df_description_train.shape)\n",
    "df_description_train.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-07T08:06:05.896420Z",
     "start_time": "2020-08-07T08:06:05.854532Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "photo_flickr_id:  40155\n",
      "album_id:  8031\n",
      "worker_id:  2176\n",
      "photo_order_in_story:  5\n"
     ]
    }
   ],
   "source": [
    "print(\"photo_flickr_id: \", len(df_description_train[\"photo_flickr_id\"].unique()))\n",
    "print(\"album_id: \", len(df_description_train[\"album_id\"].unique()))\n",
    "print(\"worker_id: \", len(df_description_train[\"worker_id\"].unique()))\n",
    "print(\"photo_order_in_story: \", len(df_description_train[\"photo_order_in_story\"].unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-07T08:06:06.474874Z",
     "start_time": "2020-08-07T08:06:06.431989Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Story album_id와 Desc album_id 비교\n",
    "\n",
    "unique_story_album_id = df_story_train[\"album_id\"].unique()\n",
    "unique_desc_album_id = df_description_train[\"album_id\"].unique()\n",
    "\n",
    "np.array_equal(unique_story_album_id, unique_desc_album_id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  Description 3개, Story 5개 (original_text 사용)\n",
    "\n",
    "<br> Album 8,031개 X Desc 3개  X 설명 5개 = 120,465개\n",
    "<br> Album 8,031개 X Story 5개 X 설명 5개 = 200,775개\n",
    "<br> Album 8,031개 X Desc 3개  X Story 5개 X 설명 5개 = 602,325개"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-07T08:08:55.631551Z",
     "start_time": "2020-08-07T08:06:08.040686Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>album_id</th>\n",
       "      <th>seq_num</th>\n",
       "      <th>desc</th>\n",
       "      <th>story</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>72157605930515606</td>\n",
       "      <td>0</td>\n",
       "      <td>Large tree with many outstretching branches an...</td>\n",
       "      <td>Our landmark tree in town was about to be dest...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>72157605930515606</td>\n",
       "      <td>1</td>\n",
       "      <td>A green sign is describing a historic tree and...</td>\n",
       "      <td>So we decided to take the day to go out and en...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>72157605930515606</td>\n",
       "      <td>2</td>\n",
       "      <td>A large tree with roots that look like crocodi...</td>\n",
       "      <td>To see the final glimpse of the roots, extendi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>72157605930515606</td>\n",
       "      <td>3</td>\n",
       "      <td>Big old tree being photographed on a sunny day</td>\n",
       "      <td>And its magnificent trunk, larger than life it...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>72157605930515606</td>\n",
       "      <td>4</td>\n",
       "      <td>Huge brown tree roots rose above the ground.</td>\n",
       "      <td>One last picture of its beauty so we could cap...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>72157605930515606</td>\n",
       "      <td>0</td>\n",
       "      <td>Large tree with many outstretching branches an...</td>\n",
       "      <td>We found this tree when we were walking in a n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>72157605930515606</td>\n",
       "      <td>1</td>\n",
       "      <td>A green sign is describing a historic tree and...</td>\n",
       "      <td>It turns out it is a popular attraction here.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>72157605930515606</td>\n",
       "      <td>2</td>\n",
       "      <td>A large tree with roots that look like crocodi...</td>\n",
       "      <td>The tree is very unusual, with its roots exposed.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>72157605930515606</td>\n",
       "      <td>3</td>\n",
       "      <td>Big old tree being photographed on a sunny day</td>\n",
       "      <td>The trunk was really wide, as much as 12 feet!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>72157605930515606</td>\n",
       "      <td>4</td>\n",
       "      <td>Huge brown tree roots rose above the ground.</td>\n",
       "      <td>You can see how big these roots are - pretty a...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            album_id  seq_num  \\\n",
       "0  72157605930515606        0   \n",
       "1  72157605930515606        1   \n",
       "2  72157605930515606        2   \n",
       "3  72157605930515606        3   \n",
       "4  72157605930515606        4   \n",
       "5  72157605930515606        0   \n",
       "6  72157605930515606        1   \n",
       "7  72157605930515606        2   \n",
       "8  72157605930515606        3   \n",
       "9  72157605930515606        4   \n",
       "\n",
       "                                                desc  \\\n",
       "0  Large tree with many outstretching branches an...   \n",
       "1  A green sign is describing a historic tree and...   \n",
       "2  A large tree with roots that look like crocodi...   \n",
       "3     Big old tree being photographed on a sunny day   \n",
       "4       Huge brown tree roots rose above the ground.   \n",
       "5  Large tree with many outstretching branches an...   \n",
       "6  A green sign is describing a historic tree and...   \n",
       "7  A large tree with roots that look like crocodi...   \n",
       "8     Big old tree being photographed on a sunny day   \n",
       "9       Huge brown tree roots rose above the ground.   \n",
       "\n",
       "                                               story  \n",
       "0  Our landmark tree in town was about to be dest...  \n",
       "1  So we decided to take the day to go out and en...  \n",
       "2  To see the final glimpse of the roots, extendi...  \n",
       "3  And its magnificent trunk, larger than life it...  \n",
       "4  One last picture of its beauty so we could cap...  \n",
       "5  We found this tree when we were walking in a n...  \n",
       "6      It turns out it is a popular attraction here.  \n",
       "7  The tree is very unusual, with its roots exposed.  \n",
       "8     The trunk was really wide, as much as 12 feet!  \n",
       "9  You can see how big these roots are - pretty a...  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "    album 기준으로 잡고\n",
    "    Desc 3개 / Story 5개\n",
    "    D1 - S1, S2, S3, S4, S5 // 5 : 5, 5, 5, 5, 5 = 5 * 5\n",
    "    D2 - S1, S2, S3, S4, S5 // 5 : 5, 5, 5, 5, 5 = 5 * 5\n",
    "    D3 - S1, S2, S3, S4, S5 // 5 : 5, 5, 5, 5, 5 = 5 * 5\n",
    "    으로 Dataframe 생성 // album 하나에 75개 생성\n",
    "\"\"\"\n",
    "total_text_list = []\n",
    "desc_length = 3\n",
    "story_length = 5\n",
    "\n",
    "for idx, album_id in enumerate(unique_story_album_id):\n",
    "    # print(\"idx: \", idx, end = \"\\r\")\n",
    "    current_desc = df_description_train[df_description_train[\"album_id\"] == album_id]\n",
    "    current_story = df_story_train[df_story_train[\"album_id\"] == album_id]\n",
    "    \n",
    "    # Desc를 기준으로\n",
    "    for desc_idx in range(desc_length):\n",
    "        desc_text_list = current_desc.iloc[desc_idx * 5 : desc_idx * 5 + 5][\"original_text\"].values\n",
    "        for story_idx in range(story_length):\n",
    "            story_text_list = current_story.iloc[story_idx * 5 : story_idx * 5 + 5][\"original_text\"].values\n",
    "            for zip_idx, (desc, story) in enumerate(zip(desc_text_list, story_text_list)):\n",
    "                total_text_list.append([album_id, zip_idx, desc, story])\n",
    "                \n",
    "desc_story_data_df = pd.DataFrame(total_text_list, columns = [\"album_id\", \"seq_num\", \"desc\", \"story\"])\n",
    "desc_story_data_df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-07T07:35:06.656901Z",
     "start_time": "2020-08-07T07:35:06.407568Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# desc_story_data_df.to_pickle(\"..\\\\..\\\\data\\\\desc_story_data_df.pkl\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}