data_preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /Users/akest/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import swifter\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from process import preproccessing\n",
    "import pickle\n",
    "import warnings\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "nltk.download('stopwords')\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('posts.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>post_id</th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "      <th>date</th>\n",
       "      <th>views_count</th>\n",
       "      <th>comments_count</th>\n",
       "      <th>bookmarks_count</th>\n",
       "      <th>rating</th>\n",
       "      <th>author_nickname</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>365293</td>\n",
       "      <td>LinkedIn автоматизировал добавление в профиль ...</td>\n",
       "      <td>Разработчики LinkedIn объявили о появившейся в...</td>\n",
       "      <td>2015-03-26 16:32:00</td>\n",
       "      <td>7300</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>+8</td>\n",
       "      <td>Teachbase</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>5005</td>\n",
       "      <td>MobileFaker: фальшивые звонки как социальное о...</td>\n",
       "      <td>Со своего мобильника вы заказываете фальшивый ...</td>\n",
       "      <td>2007-01-26 14:39:00</td>\n",
       "      <td>315</td>\n",
       "      <td>17</td>\n",
       "      <td>2</td>\n",
       "      <td>+16</td>\n",
       "      <td>alizar</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>125101</td>\n",
       "      <td>Jelastic is the new kind of Java hosting platform</td>\n",
       "      <td>Jelastic — облачный хостинг для ранее разработ...</td>\n",
       "      <td>2011-07-28 16:40:00</td>\n",
       "      <td>1300</td>\n",
       "      <td>34</td>\n",
       "      <td>17</td>\n",
       "      <td>+29</td>\n",
       "      <td>sirus</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>375301</td>\n",
       "      <td>В Mozilla работают над версией Firefox OS для ...</td>\n",
       "      <td>\\n\\nFirefox OS — относительно новая операционн...</td>\n",
       "      <td>2015-01-09 14:56:00</td>\n",
       "      <td>4800</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>+8</td>\n",
       "      <td>marks</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>130105</td>\n",
       "      <td>Как я спам слал</td>\n",
       "      <td>Введение\\nВ этой небольшой статье я хочу расск...</td>\n",
       "      <td>2011-10-10 18:35:00</td>\n",
       "      <td>25500</td>\n",
       "      <td>13</td>\n",
       "      <td>26</td>\n",
       "      <td>+19</td>\n",
       "      <td>Nike01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>232122</th>\n",
       "      <td>232122</td>\n",
       "      <td>125096</td>\n",
       "      <td>Как научить Wordpress движок дружить с Proxy? ...</td>\n",
       "      <td>Я занимаюсь разработкой локального корпоративн...</td>\n",
       "      <td>2011-07-28 11:12:00</td>\n",
       "      <td>3800</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>–1</td>\n",
       "      <td>9k1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>232123</th>\n",
       "      <td>232123</td>\n",
       "      <td>125097</td>\n",
       "      <td>Опубликованы обучающие материалы по Visual Stu...</td>\n",
       "      <td>\\nПривет, Хабр! \\n\\nБуквально вчера мы опублик...</td>\n",
       "      <td>2011-07-28 11:17:00</td>\n",
       "      <td>11200</td>\n",
       "      <td>15</td>\n",
       "      <td>19</td>\n",
       "      <td>+14</td>\n",
       "      <td>Parnassus</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>232124</th>\n",
       "      <td>232124</td>\n",
       "      <td>125098</td>\n",
       "      <td>PostSharp. Отложенная загрузка зависимостей</td>\n",
       "      <td>Кусок кода, представленный ниже, вы наверняка ...</td>\n",
       "      <td>2011-07-28 11:47:00</td>\n",
       "      <td>1700</td>\n",
       "      <td>13</td>\n",
       "      <td>17</td>\n",
       "      <td>+4</td>\n",
       "      <td>sidristij</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>232125</th>\n",
       "      <td>232125</td>\n",
       "      <td>125099</td>\n",
       "      <td>Обзор клавиатур для контроллера PS3</td>\n",
       "      <td>В принципе, накладные клавиатуры для игровых к...</td>\n",
       "      <td>2011-07-28 13:13:00</td>\n",
       "      <td>1800</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>+1</td>\n",
       "      <td>IrinaOcean</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>232126</th>\n",
       "      <td>232126</td>\n",
       "      <td>125100</td>\n",
       "      <td>Сегодня 28.07 Онлайн разбираем на винтики: iOs...</td>\n",
       "      <td>UPD онлайн трансляция закончилась, всем спасиб...</td>\n",
       "      <td>2011-07-28 12:15:00</td>\n",
       "      <td>4300</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>+10</td>\n",
       "      <td>Artkumpel</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>232127 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Unnamed: 0  post_id  \\\n",
       "0                0   365293   \n",
       "1                1     5005   \n",
       "2                2   125101   \n",
       "3                3   375301   \n",
       "4                4   130105   \n",
       "...            ...      ...   \n",
       "232122      232122   125096   \n",
       "232123      232123   125097   \n",
       "232124      232124   125098   \n",
       "232125      232125   125099   \n",
       "232126      232126   125100   \n",
       "\n",
       "                                                    title  \\\n",
       "0       LinkedIn автоматизировал добавление в профиль ...   \n",
       "1       MobileFaker: фальшивые звонки как социальное о...   \n",
       "2       Jelastic is the new kind of Java hosting platform   \n",
       "3       В Mozilla работают над версией Firefox OS для ...   \n",
       "4                                         Как я спам слал   \n",
       "...                                                   ...   \n",
       "232122  Как научить Wordpress движок дружить с Proxy? ...   \n",
       "232123  Опубликованы обучающие материалы по Visual Stu...   \n",
       "232124        PostSharp. Отложенная загрузка зависимостей   \n",
       "232125                Обзор клавиатур для контроллера PS3   \n",
       "232126  Сегодня 28.07 Онлайн разбираем на винтики: iOs...   \n",
       "\n",
       "                                                     text  \\\n",
       "0       Разработчики LinkedIn объявили о появившейся в...   \n",
       "1       Со своего мобильника вы заказываете фальшивый ...   \n",
       "2       Jelastic — облачный хостинг для ранее разработ...   \n",
       "3       \\n\\nFirefox OS — относительно новая операционн...   \n",
       "4       Введение\\nВ этой небольшой статье я хочу расск...   \n",
       "...                                                   ...   \n",
       "232122  Я занимаюсь разработкой локального корпоративн...   \n",
       "232123  \\nПривет, Хабр! \\n\\nБуквально вчера мы опублик...   \n",
       "232124  Кусок кода, представленный ниже, вы наверняка ...   \n",
       "232125  В принципе, накладные клавиатуры для игровых к...   \n",
       "232126  UPD онлайн трансляция закончилась, всем спасиб...   \n",
       "\n",
       "                       date  views_count  comments_count  bookmarks_count  \\\n",
       "0       2015-03-26 16:32:00         7300               1               10   \n",
       "1       2007-01-26 14:39:00          315              17                2   \n",
       "2       2011-07-28 16:40:00         1300              34               17   \n",
       "3       2015-01-09 14:56:00         4800               3                6   \n",
       "4       2011-10-10 18:35:00        25500              13               26   \n",
       "...                     ...          ...             ...              ...   \n",
       "232122  2011-07-28 11:12:00         3800               5                6   \n",
       "232123  2011-07-28 11:17:00        11200              15               19   \n",
       "232124  2011-07-28 11:47:00         1700              13               17   \n",
       "232125  2011-07-28 13:13:00         1800              18                1   \n",
       "232126  2011-07-28 12:15:00         4300               3                6   \n",
       "\n",
       "       rating author_nickname  \n",
       "0          +8       Teachbase  \n",
       "1         +16          alizar  \n",
       "2         +29           sirus  \n",
       "3          +8           marks  \n",
       "4         +19          Nike01  \n",
       "...       ...             ...  \n",
       "232122     –1             9k1  \n",
       "232123    +14       Parnassus  \n",
       "232124     +4       sidristij  \n",
       "232125     +1      IrinaOcean  \n",
       "232126    +10       Artkumpel  \n",
       "\n",
       "[232127 rows x 10 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 232127 entries, 0 to 232126\n",
      "Data columns (total 10 columns):\n",
      " #   Column           Non-Null Count   Dtype \n",
      "---  ------           --------------   ----- \n",
      " 0   Unnamed: 0       232127 non-null  int64 \n",
      " 1   post_id          232127 non-null  int64 \n",
      " 2   title            232127 non-null  object\n",
      " 3   text             231467 non-null  object\n",
      " 4   date             232127 non-null  object\n",
      " 5   views_count      232127 non-null  int64 \n",
      " 6   comments_count   232127 non-null  int64 \n",
      " 7   bookmarks_count  232127 non-null  int64 \n",
      " 8   rating           232127 non-null  object\n",
      " 9   author_nickname  232120 non-null  object\n",
      "dtypes: int64(5), object(5)\n",
      "memory usage: 17.7+ MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Дропнем строки с пропущенными значениями "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Оставим только нужные столбцы"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[['title', 'text', 'rating']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Переведем рейтинг в [0, 1]. Отсортируем по нему"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['rating'] = pd.to_numeric(df['rating'].apply(lambda s: s.replace('–', '-')))\n",
    "df['rating'] = df['rating'] + np.abs(min(df['rating']))\n",
    "df['rating'] = df['rating'] / max(df['rating'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.sort_values('rating', ascending=False, ignore_index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Тут совсем много строк для данных с таким большим объемом текста(статьи). Я даже с параллельной обработкой буду ждать очень долго. Поэтому я сокращу их, чтобы получилось чуть больше 100 мб. Я решил взять 10000 самых рейтинговых статей."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Делаем приватный монитор из старого LCD монитора</td>\n",
       "      <td>\\n\\nВы наконец-то можете сделать кое-что со св...</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Были получены исходники 3300 глобальных интерн...</td>\n",
       "      <td>Пару месяцев назад нами (2Товарища и Антон Иса...</td>\n",
       "      <td>0.819742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>История игрушки. Поле Чудес</td>\n",
       "      <td>Случилось это в городе, закрытом от шпионов, ц...</td>\n",
       "      <td>0.671980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[Обновлено в 10:52, 14.12.19] В офисе Nginx пр...</td>\n",
       "      <td>Другие материалы по теме:\\nEng version\\nЧто зн...</td>\n",
       "      <td>0.597180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Как Денис Крючков выкупил Хабр у Mail.ru</td>\n",
       "      <td>Денис Крючков (deniskin) рассказал Roem.ru о т...</td>\n",
       "      <td>0.591662</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "0   Делаем приватный монитор из старого LCD монитора   \n",
       "1  Были получены исходники 3300 глобальных интерн...   \n",
       "2                        История игрушки. Поле Чудес   \n",
       "3  [Обновлено в 10:52, 14.12.19] В офисе Nginx пр...   \n",
       "4           Как Денис Крючков выкупил Хабр у Mail.ru   \n",
       "\n",
       "                                                text    rating  \n",
       "0  \\n\\nВы наконец-то можете сделать кое-что со св...  1.000000  \n",
       "1  Пару месяцев назад нами (2Товарища и Антон Иса...  0.819742  \n",
       "2  Случилось это в городе, закрытом от шпионов, ц...  0.671980  \n",
       "3  Другие материалы по теме:\\nEng version\\nЧто зн...  0.597180  \n",
       "4  Денис Крючков (deniskin) рассказал Roem.ru о т...  0.591662  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.head(10000)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Предобработаем текстовые данные. Хотелось бы ускорить обработку. Для этого решил использовать swifter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0949b48a36534403bb533bb8d4ba3916",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Pandas Apply:   0%|          | 0/10000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "580d486722204b1988690217f53afebe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Pandas Apply:   0%|          | 0/10000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df['preproc_text'] = df['text'].swifter.apply(preproccessing)\n",
    "df['preproc_title'] = df['title'].swifter.apply(preproccessing)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Векторизуем наши текстовые данные"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TfidfVectorizer()"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer = TfidfVectorizer()\n",
    "vectorizer.fit(df.preproc_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TfidfVectorizer()"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer_title = TfidfVectorizer()\n",
    "vectorizer_title.fit(df.preproc_title)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Запишем их с помощью pickle в файл"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open('pickled', 'wb')\n",
    "data = {'titles': df['title'].to_list(),\n",
    "        'ratings': df['rating'].to_list(), 'text_model': vectorizer, 'texts': df['text'].to_list(),\n",
    "       'texts_processed': df['preproc_text'].to_list(), 'titles_processed': df['preproc_title'].to_list(),\n",
    "       'title_model': vectorizer_title}\n",
    "pickle.dump(data, f)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}