Preprocessing_corpus.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Preprocessing_corpus.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyNYCozuU1RCsHo1Jqd61IkW",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Adrok24/Proyecto-Integrador/blob/branch_1/Preprocessing_corpus.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bv2cvuyxqGWW"
      },
      "source": [
        "# Importaciones y descargas"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8y4qcvdI9FYt",
        "outputId": "c648681c-fd27-439c-f18b-d3efa2eb73f5"
      },
      "source": [
        "from google.colab import files\r\n",
        "from google.colab import drive\r\n",
        "import os\r\n",
        "from os.path import isfile, join\r\n",
        "import pandas as pd\r\n",
        "import re\r\n",
        "from nltk.tokenize import sent_tokenize, word_tokenize\r\n",
        "import nltk\r\n",
        "nltk.download('punkt')\r\n",
        "pd.options.display.max_colwidth = 150"
      ],
      "execution_count": 117,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "u5BuSqpHrYUO",
        "outputId": "6f492520-af1c-4e49-ca86-67c6604da0cd"
      },
      "source": [
        "!pip install nltk"
      ],
      "execution_count": 118,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: nltk in /usr/local/lib/python3.6/dist-packages (3.2.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from nltk) (1.15.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Iopqukas9yGN",
        "outputId": "1e99d730-4d90-43ba-b96f-b138f01ddd1b"
      },
      "source": [
        "!unrar x '/content/drive/My Drive/Colab Notebooks/desafio final/txt_books_spa.rar'"
      ],
      "execution_count": 119,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "UNRAR 5.50 freeware      Copyright (c) 1993-2017 Alexander Roshal\n",
            "\n",
            "\n",
            "Extracting from /content/drive/My Drive/Colab Notebooks/desafio final/txt_books_spa.rar\n",
            "\n",
            "\n",
            "Would you like to replace the existing file txt_books_spa/txt_books_spa/cortazar txt/alvo_el_crepusculo_-_Julio_Cortazar.txt\n",
            "198858 bytes, modified on 2021-02-02 23:32\n",
            "with a new one\n",
            "198858 bytes, modified on 2021-02-02 23:32\n",
            "\n",
            "[Y]es, [N]o, [A]ll, n[E]ver, [R]ename, [Q]uit q\n",
            "\n",
            "Program aborted\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HkXZb01PwDWl",
        "outputId": "aca1ebfa-2d63-4404-c469-e754531a80a0"
      },
      "source": [
        "drive.mount('/content/drive')"
      ],
      "execution_count": 126,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KAvIDuCxrCpt"
      },
      "source": [
        "# Generar una lista de nombres"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xoqq5Ai_Z9pb"
      },
      "source": [
        "folder = '/content/drive/My Drive/Colab Notebooks/desafio final/'\r\n",
        "\r\n",
        "path_to_names = os.path.join(folder, 'list_names.csv')\r\n",
        "data_names = pd.read_csv(path_to_names)"
      ],
      "execution_count": 121,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h86Imc7EmDrc"
      },
      "source": [
        "list_names = data_names.Name.drop_duplicates().tolist()"
      ],
      "execution_count": 150,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CJNhNV4jSTHs"
      },
      "source": [
        "delete_from_Names = ['Donde','Por','Para','Pero', 'Al','Sin','Con','Por', 'De','Desde','Por','Para','Hasta','Contra','El','Los','Las','Sus','Esas','Esos','Todos','Mis','Tus', 'Ese', 'Aquel', 'Esos', 'Ellos', 'En', 'Me', 'Te', 'Se', 'Le', 'Ve']"
      ],
      "execution_count": 151,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kIAbTb_yRqP6",
        "outputId": "81c58ac7-b5a4-4e71-8cf2-3436a94c4a86"
      },
      "source": [
        "len(list_names)"
      ],
      "execution_count": 152,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "43675"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 152
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yu_5WQKpSBEB",
        "outputId": "9487f677-5ffd-4917-a961-72c1deef5c26"
      },
      "source": [
        "for x in delete_from_Names:\r\n",
        "  if (x in list_names):\r\n",
        "    list_names.remove(x)\r\n",
        "\r\n",
        "len(list_names)"
      ],
      "execution_count": 154,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "43663"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 154
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6-TM32pMrIOf"
      },
      "source": [
        "# Generacion de un listado de libros"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "irOhibDlBq_t",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "84b79711-6cdd-482e-8b2f-2f84cccb1c15"
      },
      "source": [
        "mypath = '/content/txt_books_spa' \r\n",
        "\r\n",
        "# list_files = [join(root, name_file) for root, dirs, name_file in os.walk(mypath, topdown=False)] \r\n",
        "\r\n",
        "count_files = 0\r\n",
        "dict_paths = {}\r\n",
        "for root, dirs, files in os.walk(mypath, topdown=False):\r\n",
        "    for name_file in files:\r\n",
        "        dict_paths[count_files] = {\"path\": os.path.join(root, name_file), \"name\": name_file.replace('.txt', ''), \"genre\": root.split('/')[4].split(' ')[0], \"category\" : \"Book\"}\r\n",
        "        count_files = count_files + 1\r\n",
        "\r\n",
        "df_path = pd.DataFrame.from_dict(dict_paths, \"index\")\r\n",
        "display(df_path.shape)"
      ],
      "execution_count": 155,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "(154, 4)"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OnqabupFrfzP"
      },
      "source": [
        "# Separacion en lineas"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "plZ01OetJU6u"
      },
      "source": [
        "def get_lines(line, book, dict_book_final, count_lines):\r\n",
        "  if (len(line) > 130):\r\n",
        "    split_line = line.split('.')\r\n",
        "    if (len(split_line) == 1):\r\n",
        "      split_line = line.split(',')\r\n",
        "    for s_line in split_line:\r\n",
        "      if ((len(s_line) < 130) and (len(s_line) > 30)):\r\n",
        "        dict_book_final[count_lines] = {\"line\": s_line, \"len\": len(s_line), \"book\": book['name'], \"genre\": book['genre'], \"category\": book['category']}\r\n",
        "        count_lines = count_lines + 1\r\n",
        "  elif (len(line) > 30):  \r\n",
        "    dict_book_final[count_lines] = {\"line\": line, \"len\": len(line), \"book\": book['name'], \"genre\": book['genre'], \"category\": book['category']}\r\n",
        "    count_lines = count_lines + 1\r\n",
        "  return dict_book_final, count_lines"
      ],
      "execution_count": 156,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y3U8HCyk6Bed",
        "outputId": "cccdbd6c-b03d-4bc6-ca7e-f4a7de80b38d"
      },
      "source": [
        "dict_book_final = {}\r\n",
        "count_lines = 0\r\n",
        "for index, book in df_path.iterrows():\r\n",
        "  corpus = open(book['path'], \"r\")\r\n",
        "  cadena = corpus.read()\r\n",
        "  list_cadena = sent_tokenize(cadena, language='spanish')\r\n",
        "  print('cant sent:',len(list_cadena),book['name'])\r\n",
        "\r\n",
        "  for line in list_cadena:\r\n",
        "    if (line.find('\\n') == -1):\r\n",
        "      dict_book_final, count_lines = get_lines(line, book, dict_book_final, count_lines)\r\n",
        "    else:\r\n",
        "      split_line = line.split('\\n')\r\n",
        "      for s_line in split_line:\r\n",
        "        dict_book_final, count_lines = get_lines(s_line, book, dict_book_final, count_lines)"
      ],
      "execution_count": 157,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cant sent: 1768 Bestiario_-_Julio_Cortazar\n",
            "cant sent: 950 mbradas_culturales_-_Julio_Cortazar\n",
            "cant sent: 788 Un_tal_Lucas_-_Julio_Cortazar\n",
            "cant sent: 910 Obra_critica_I_-_Julio_Cortazar\n",
            "cant sent: 11572 Cartas_1937-1954_-_Julio_Cortazar\n",
            "cant sent: 8700 Los-premios-Julio-Cortazar\n",
            "cant sent: 2679 ases_de_literatura_-_Julio_Cortazar\n",
            "cant sent: 2328 Juegos-Julio-Cortazar\n",
            "cant sent: 12746 Cartas_1955-1964_-_Julio_Cortazar\n",
            "cant sent: 2711 Los-autonautas-de-la-cosmopista-Julio-Cortazar\n",
            "cant sent: 162 Silvalandia_-_Julio_Cortazar\n",
            "cant sent: 3665 Obra_critica_II_-_Julio_Cortazar\n",
            "cant sent: 1187 Deshoras_-_Julio_Cortazar\n",
            "cant sent: 2213 on_y_otros_relatos_-_Julio_Cortazar\n",
            "cant sent: 1472 Nicaragua-tan-violentamente-dulce-Julio-Cortazar\n",
            "cant sent: 1185 alvo_el_crepusculo_-_Julio_Cortazar\n",
            "cant sent: 2322 Las-armas-secretas-Julio-Cortazar\n",
            "cant sent: 1057 mos_tanto_a_Glenda_-_Julio_Cortazar\n",
            "cant sent: 3042 Pasajes_-_Julio_Cortazar\n",
            "cant sent: 4446 or_y_otros_relatos_-_Julio_Cortazar\n",
            "cant sent: 2161 Obra_critica_III_-_Julio_Cortazar\n",
            "cant sent: 2584 _Modelo_para_armar_-_Julio_Cortazar\n",
            "cant sent: 547 Los-reyes-Julio-Cortazar\n",
            "cant sent: 1564 La-otra-orilla-Julio-Cortazar\n",
            "cant sent: 1628 n_que_anda_por_ahi_-_Julio_Cortazar\n",
            "cant sent: 935 Octaedro_-_Julio_Cortazar\n",
            "cant sent: 1643 os_fuegos_el_fuego_-_Julio_Cortazar\n",
            "cant sent: 3664 l_miedo_a_la_libertad_-_Erich_Fromm\n",
            "cant sent: 1016 La_mano_invisible_-_Adam_Smith\n",
            "cant sent: 555 e_los_griegos_-_Friedrich_Nietzsche\n",
            "cant sent: 1388 La_gran_moral_-_Aristoteles\n",
            "cant sent: 2004 ente_de_haber_nacido_-_E._M._Cioran\n",
            "cant sent: 7389 lo_Zaratustra_-_Friedrich_Nietzsche\n",
            "cant sent: 970 El_Anticristo_-_Friedrich_Nietzsche\n",
            "cant sent: 1574 as_bellas_artes_-_Daniel_Innerarity\n",
            "cant sent: 4064 ielo_-_Meteorologicos_-_Aristoteles\n",
            "cant sent: 1713 tica_de_urgencia_-_Fernando_Savater\n",
            "cant sent: 2201 o_en_un_solo_atomo_-_Dalai_Lama_XIV\n",
            "cant sent: 1674 Acerca_del_alma_-_Aristoteles\n",
            "cant sent: 1441 a_caida_en_el_tiempo_-_E._M._Cioran\n",
            "cant sent: 1080 idad_e_individuo_-_Bertrand_Russell\n",
            "cant sent: 2910 l_budismo_-_Ernest_Yassine_Bendriss\n",
            "cant sent: 1267 ogia_del_sofista_-_Fernando_Savater\n",
            "cant sent: 2346 _cosas_que_escribi_-_Ainhoa_Escarti\n",
            "cant sent: 64 el_desastre_-_Leopoldo_Maria_Panero\n",
            "cant sent: 35 izofrenicos_-_Leopoldo_Maria_Panero\n",
            "cant sent: 2259 Vetas_profundas_-_Fernando_Aramburu\n",
            "cant sent: 64 e_la_locura_-_Leopoldo_Maria_Panero\n",
            "cant sent: 226 ado_de_exilio_-_Cristina_Peri_Rossi\n",
            "cant sent: 18 onversacion_-_Leopoldo_Maria_Panero\n",
            "cant sent: 325 ntologia_poetica_-_Antonio_Gamoneda\n",
            "cant sent: 805 o_de_una_chica_en_llamas_-_SrtaBebi\n",
            "cant sent: 490 Seleccion_de_poemas_-_Adonis\n",
            "cant sent: 36 Abismo_-_Leopoldo_Maria_Panero\n",
            "cant sent: 78 _la_lampara_-_Leopoldo_Maria_Panero\n",
            "cant sent: 1076 Amor_y_asco_-_SrtaBebi\n",
            "cant sent: 91 a_del_miedo_-_Leopoldo_Maria_Panero\n",
            "cant sent: 727 aricias_al_tiempo_-_Diego_Fortunato\n",
            "cant sent: 344 42_poemas_-_Fernando_Pessoa\n",
            "cant sent: 60 Sombra_-_Leopoldo_Maria_Panero\n",
            "cant sent: 935 _frente_al_espejo_-_Denis_Hernandez\n",
            "cant sent: 828 _al_viento_Poemas_-_Diego_Fortunato\n",
            "cant sent: 330 as_cuatro_estaciones_-_Basho_Matsuo\n",
            "cant sent: 3744 _como_fueron_-_Eloy_Sanchez_Rosillo\n",
            "cant sent: 274 uaderno_del_delirio_-_Elvira_Daudet\n",
            "cant sent: 329 cion_de_mujer_-_Cristina_Peri_Rossi\n",
            "cant sent: 350 Susurros_del_Alma_-_Alec_Blackhell\n",
            "cant sent: 544 _versos_perdidos_-_Txus_di_Fellatio\n",
            "cant sent: 162 Aquella_noche_-_Cristina_Peri_Rossi\n",
            "cant sent: 283 _medianamente_dulces_-_Alma_Phillip\n",
            "cant sent: 408 Charles_Bukowski_-_Charles_Bukowski\n",
            "cant sent: 505 bertad_y_sangre_-_Manolillo_Chinato\n",
            "cant sent: 377 _grandes_mujeres_-_Alfonsina_Storni\n",
            "cant sent: 2195 as_llaves_del_infierno_-_A._Rolcest\n",
            "cant sent: 8379 do_late_un_corazon_-_Danielle_Steel\n",
            "cant sent: 2523 _No_habra_retirada_-_A._Rolcest\n",
            "cant sent: 4001 asion_bajo_el_hielo_-_Allison_Leigh\n",
            "cant sent: 3908 a_extrana_propuesta_-_Allison_Leigh\n",
            "cant sent: 2174 La_soga_y_el_colt_-_A._Rolcest\n",
            "cant sent: 9269 Deseos_concedidos_-_Danielle_Steel\n",
            "cant sent: 9917 Destinos_errantes_-_Danielle_Steel\n",
            "cant sent: 8175 El_viaje_-_Danielle_Steel\n",
            "cant sent: 9434 Dulce_y_amargo_-_Danielle_Steel\n",
            "cant sent: 4330 amorado_de_la_novia_-_Allison_Leigh\n",
            "cant sent: 2224 Buitres_de_la_pradera_-_A._Rolcest\n",
            "cant sent: 3643 rapados_en_la_nieve_-_Allison_Leigh\n",
            "cant sent: 5262 El_regalo_-_Danielle_Steel\n",
            "cant sent: 2255 Orden_de_Invasion_-_A._Rolcest\n",
            "cant sent: 2174 La_captura_de_la_fiera_-_A._Rolcest\n",
            "cant sent: 4264 Regreso_a_casa_-_Allison_Leigh\n",
            "cant sent: 10277 Amor_y_lagrimas_-_Danielle_Steel\n",
            "cant sent: 9322 El_beso_-_Danielle_Steel\n",
            "cant sent: 2405 Guarida_de_condenados_-_A._Rolcest\n",
            "cant sent: 8846 Accidente_-_Danielle_Steel\n",
            "cant sent: 2178 La_ofrenda_del_diablo_-_A._Rolcest\n",
            "cant sent: 2799 El_pistolero_del_norte_-_A._Rolcest\n",
            "cant sent: 2777 La_tumba_espera_-_A._Rolcest\n",
            "cant sent: 7410 suntos_del_corazon_-_Danielle_Steel\n",
            "cant sent: 2241 Tierra_de_vencidos_-_A._Rolcest\n",
            "cant sent: 1873 uella_de_los_cuatreros_-_A._Rolcest\n",
            "cant sent: 2455 Peor_que_fieras_-_A._Rolcest\n",
            "cant sent: 6841 El_legado_-_Danielle_Steel\n",
            "cant sent: 9742 El_anillo_-_Danielle_Steel\n",
            "cant sent: 2021 Brigada_de_choque_-_A._Rolcest\n",
            "cant sent: 2468 infierno_de_los_idolos_-_A._Rolcest\n",
            "cant sent: 2328 Jaque_de_rufianes_-_A._Rolcest\n",
            "cant sent: 10090 Aguila_solitaria_-_Danielle_Steel\n",
            "cant sent: 5756 Cuento_de_hadas_-_Danielle_Steel\n",
            "cant sent: 8279 Calidoscopio_-_Danielle_Steel\n",
            "cant sent: 4294 trato_de_matrimonio_-_Allison_Leigh\n",
            "cant sent: 2339 azote_de_los_cuatreros_-_A._Rolcest\n",
            "cant sent: 2567 Cerco_de_metralletas_-_A._Rolcest\n",
            "cant sent: 2648 _Fuera_los_rufianes_-_A._Rolcest\n",
            "cant sent: 2828 La_caravana_muerta_-_A._Rolcest\n",
            "cant sent: 2453 El_naipe_revancha_-_A._Rolcest\n",
            "cant sent: 4271 inco_dias_en_Paris_-_Danielle_Steel\n",
            "cant sent: 2684 El_tigre_sediento_-_A._Rolcest\n",
            "cant sent: 1921 Cerco_a_Damasco_-_A._Rolcest\n",
            "cant sent: 1997 Domando_horizontes_-_A._Rolcest\n",
            "cant sent: 6657 honor_del_silencio_-_Danielle_Steel\n",
            "cant sent: 10138 El_precio_del_amor_-_Danielle_Steel\n",
            "cant sent: 8948 Ecos_-_Danielle_Steel\n",
            "cant sent: 4542 romesas_del_corazon_-_Allison_Leigh\n",
            "cant sent: 10322 ora_y_para_siempre_-_Danielle_Steel\n",
            "cant sent: 3853 quero_y_la_heredera_-_Allison_Leigh\n",
            "cant sent: 10230 no_de_una_estrella_-_Danielle_Steel\n",
            "cant sent: 2205 Hoguera_en_la_noche_-_A._Rolcest\n",
            "cant sent: 3443 Acto_de_fe_-_Danielle_Steel\n",
            "cant sent: 8074 argo_camino_a_casa_-_Danielle_Steel\n",
            "cant sent: 9555 El_rancho_-_Danielle_Steel\n",
            "cant sent: 3036 La_meta_del_odio_-_A._Rolcest\n",
            "cant sent: 2386 Sin_testigos_-_A._Rolcest\n",
            "cant sent: 2703 La_muerte_invita_-_A._Rolcest\n",
            "cant sent: 2430 Consigna_plomo_-_A._Rolcest\n",
            "cant sent: 2421 Linea_de_fuego_-_A._Rolcest\n",
            "cant sent: 2080 Hampa_brillante_-_A._Rolcest\n",
            "cant sent: 2298 Disparos_sin_respuesta_-_A._Rolcest\n",
            "cant sent: 2336 Empujado_a_la_muerte_-_A._Rolcest\n",
            "cant sent: 4010 El_baile_del_amor_-_Allison_Leigh\n",
            "cant sent: 2307 Invitacion_a_la_muerte_-_A._Rolcest\n",
            "cant sent: 2383 Chantaje_en_las_tumbas_-_A._Rolcest\n",
            "cant sent: 2776 La_pantera_dormida_-_A._Rolcest\n",
            "cant sent: 2218 El_reto_de_la_amazona_-_A._Rolcest\n",
            "cant sent: 4214 El_clon_-_Danielle_Steel\n",
            "cant sent: 2056 La_ley_en_el_revolver_-_A._Rolcest\n",
            "cant sent: 2157 ras_quedan_los_muertos_-_A._Rolcest\n",
            "cant sent: 2247 La_espera_del_buitre_-_A._Rolcest\n",
            "cant sent: 11231 Album_de_familia_-_Danielle_Steel\n",
            "cant sent: 1848 _Colt_huracan_-_A._Rolcest\n",
            "cant sent: 7335 Blue_-_Danielle_Steel\n",
            "cant sent: 8113 El_fantasma_-_Danielle_Steel\n",
            "cant sent: 2019 La_trampa_abierta_-_A._Rolcest\n",
            "cant sent: 8148 Ansia_de_amor_-_Danielle_Steel\n",
            "cant sent: 6497 arles_street_no_44_-_Danielle_Steel\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "id": "vW9qzAQMrBSL",
        "outputId": "7a12f7ce-6dc0-490c-b5aa-edc130058bc8"
      },
      "source": [
        "df_book_lines = pd.DataFrame.from_dict(dict_book_final, \"index\")\r\n",
        "display(df_book_lines)"
      ],
      "execution_count": 158,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>line</th>\n",
              "      <th>len</th>\n",
              "      <th>book</th>\n",
              "      <th>genre</th>\n",
              "      <th>category</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Bestiario es el primer libro de relatos que Julio Cortázar publica con su auténtico nombre.</td>\n",
              "      <td>91</td>\n",
              "      <td>Bestiario_-_Julio_Cortazar</td>\n",
              "      <td>cortazar</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>No hay en estas ocho obras maestras ni el menor balbuceo ni resacas juveniles: son perfectas.</td>\n",
              "      <td>93</td>\n",
              "      <td>Bestiario_-_Julio_Cortazar</td>\n",
              "      <td>cortazar</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Sorpresa o incomodidad son, en cada texto, un condimento que se agrega al placer indescriptible de su lectura.</td>\n",
              "      <td>110</td>\n",
              "      <td>Bestiario_-_Julio_Cortazar</td>\n",
              "      <td>cortazar</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Retoque de cubierta: Un_Tal_Lucas</td>\n",
              "      <td>33</td>\n",
              "      <td>Bestiario_-_Julio_Cortazar</td>\n",
              "      <td>cortazar</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>A Paco, que gustaba de mis relatos</td>\n",
              "      <td>34</td>\n",
              "      <td>Bestiario_-_Julio_Cortazar</td>\n",
              "      <td>cortazar</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>348755</th>\n",
              "      <td>Esa dirección ya formaba parte del pasado, de otra vida que ya no existía.</td>\n",
              "      <td>74</td>\n",
              "      <td>arles_street_no_44_-_Danielle_Steel</td>\n",
              "      <td>Novelas</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>348756</th>\n",
              "      <td>Ahora le pertenecía a otra familia que la querría tanto como ella.</td>\n",
              "      <td>66</td>\n",
              "      <td>arles_street_no_44_-_Danielle_Steel</td>\n",
              "      <td>Novelas</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>348757</th>\n",
              "      <td>El 44 de Charles Street era un episodio de su vida, mucho más que una simple casa.</td>\n",
              "      <td>82</td>\n",
              "      <td>arles_street_no_44_-_Danielle_Steel</td>\n",
              "      <td>Novelas</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>348758</th>\n",
              "      <td>Aquel capítulo ya estaba cerrado.</td>\n",
              "      <td>33</td>\n",
              "      <td>arles_street_no_44_-_Danielle_Steel</td>\n",
              "      <td>Novelas</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>348759</th>\n",
              "      <td>El siguiente acababa de empezar.</td>\n",
              "      <td>32</td>\n",
              "      <td>arles_street_no_44_-_Danielle_Steel</td>\n",
              "      <td>Novelas</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>348760 rows × 5 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                                                                                  line  ...  category\n",
              "0                          Bestiario es el primer libro de relatos que Julio Cortázar publica con su auténtico nombre.  ...      Book\n",
              "1                        No hay en estas ocho obras maestras ni el menor balbuceo ni resacas juveniles: son perfectas.  ...      Book\n",
              "2       Sorpresa o incomodidad son, en cada texto, un condimento que se agrega al placer indescriptible de su lectura.  ...      Book\n",
              "3                                                                                    Retoque de cubierta: Un_Tal_Lucas  ...      Book\n",
              "4                                                                                   A Paco, que gustaba de mis relatos  ...      Book\n",
              "...                                                                                                                ...  ...       ...\n",
              "348755                                      Esa dirección ya formaba parte del pasado, de otra vida que ya no existía.  ...      Book\n",
              "348756                                              Ahora le pertenecía a otra familia que la querría tanto como ella.  ...      Book\n",
              "348757                              El 44 de Charles Street era un episodio de su vida, mucho más que una simple casa.  ...      Book\n",
              "348758                                                                               Aquel capítulo ya estaba cerrado.  ...      Book\n",
              "348759                                                                                El siguiente acababa de empezar.  ...      Book\n",
              "\n",
              "[348760 rows x 5 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HFSXW-7nrqsj"
      },
      "source": [
        "# Filtrado y preprocesamiento"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "26ozyMV5vRP9"
      },
      "source": [
        "def preprocess_sentence(sentence):\r\n",
        "  sentence = sentence.strip()\r\n",
        "  # vamos a crear espacios entre las frases y los puntos:\r\n",
        "  # ej: \"he is a boy.\" => \"he is a boy .\"\r\n",
        "  sentence = re.sub(r\"([?.!,:])\", r\" \\1 \", sentence)\r\n",
        "  sentence = re.sub(r'[\" \"]+', \" \", sentence)\r\n",
        "  sentence = re.sub(r'[A-Z]{2,20}', \" \", sentence)\r\n",
        "  sentence = re.sub(r'\\d+', \"@number\", sentence) \r\n",
        "  sentence = sentence.strip()\r\n",
        "  return sentence"
      ],
      "execution_count": 159,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f88LxhTxvzBV"
      },
      "source": [
        "df_book_lines.iloc[:,0] = df_book_lines.iloc[:,0].apply(preprocess_sentence)"
      ],
      "execution_count": 160,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "id": "NV9U-H5RwJDs",
        "outputId": "b5344b12-4cef-4eb6-d50e-4a6461f63759"
      },
      "source": [
        "df_book_lines"
      ],
      "execution_count": 161,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>line</th>\n",
              "      <th>len</th>\n",
              "      <th>book</th>\n",
              "      <th>genre</th>\n",
              "      <th>category</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Bestiario es el primer libro de relatos que Julio Cortázar publica con su auténtico nombre .</td>\n",
              "      <td>91</td>\n",
              "      <td>Bestiario_-_Julio_Cortazar</td>\n",
              "      <td>cortazar</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>No hay en estas ocho obras maestras ni el menor balbuceo ni resacas juveniles : son perfectas .</td>\n",
              "      <td>93</td>\n",
              "      <td>Bestiario_-_Julio_Cortazar</td>\n",
              "      <td>cortazar</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Sorpresa o incomodidad son , en cada texto , un condimento que se agrega al placer indescriptible de su lectura .</td>\n",
              "      <td>110</td>\n",
              "      <td>Bestiario_-_Julio_Cortazar</td>\n",
              "      <td>cortazar</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Retoque de cubierta : Un_Tal_Lucas</td>\n",
              "      <td>33</td>\n",
              "      <td>Bestiario_-_Julio_Cortazar</td>\n",
              "      <td>cortazar</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>A Paco , que gustaba de mis relatos</td>\n",
              "      <td>34</td>\n",
              "      <td>Bestiario_-_Julio_Cortazar</td>\n",
              "      <td>cortazar</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>348755</th>\n",
              "      <td>Esa dirección ya formaba parte del pasado , de otra vida que ya no existía .</td>\n",
              "      <td>74</td>\n",
              "      <td>arles_street_no_44_-_Danielle_Steel</td>\n",
              "      <td>Novelas</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>348756</th>\n",
              "      <td>Ahora le pertenecía a otra familia que la querría tanto como ella .</td>\n",
              "      <td>66</td>\n",
              "      <td>arles_street_no_44_-_Danielle_Steel</td>\n",
              "      <td>Novelas</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>348757</th>\n",
              "      <td>El @number de Charles Street era un episodio de su vida , mucho más que una simple casa .</td>\n",
              "      <td>82</td>\n",
              "      <td>arles_street_no_44_-_Danielle_Steel</td>\n",
              "      <td>Novelas</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>348758</th>\n",
              "      <td>Aquel capítulo ya estaba cerrado .</td>\n",
              "      <td>33</td>\n",
              "      <td>arles_street_no_44_-_Danielle_Steel</td>\n",
              "      <td>Novelas</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>348759</th>\n",
              "      <td>El siguiente acababa de empezar .</td>\n",
              "      <td>32</td>\n",
              "      <td>arles_street_no_44_-_Danielle_Steel</td>\n",
              "      <td>Novelas</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>348760 rows × 5 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                                                                                     line  ...  category\n",
              "0                            Bestiario es el primer libro de relatos que Julio Cortázar publica con su auténtico nombre .  ...      Book\n",
              "1                         No hay en estas ocho obras maestras ni el menor balbuceo ni resacas juveniles : son perfectas .  ...      Book\n",
              "2       Sorpresa o incomodidad son , en cada texto , un condimento que se agrega al placer indescriptible de su lectura .  ...      Book\n",
              "3                                                                                      Retoque de cubierta : Un_Tal_Lucas  ...      Book\n",
              "4                                                                                     A Paco , que gustaba de mis relatos  ...      Book\n",
              "...                                                                                                                   ...  ...       ...\n",
              "348755                                       Esa dirección ya formaba parte del pasado , de otra vida que ya no existía .  ...      Book\n",
              "348756                                                Ahora le pertenecía a otra familia que la querría tanto como ella .  ...      Book\n",
              "348757                          El @number de Charles Street era un episodio de su vida , mucho más que una simple casa .  ...      Book\n",
              "348758                                                                                 Aquel capítulo ya estaba cerrado .  ...      Book\n",
              "348759                                                                                  El siguiente acababa de empezar .  ...      Book\n",
              "\n",
              "[348760 rows x 5 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 161
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YORmaCABZ9vE",
        "outputId": "879f1cd6-2fe1-412c-e030-4cf2f5ab333e"
      },
      "source": [
        "# Enmascaramos los nombres propios, y marcamos algunas lineas que tienen caracteres especiales para luego filtrarlas\r\n",
        "\r\n",
        "from IPython.display import clear_output\r\n",
        "list_new_rows = [] \r\n",
        "for index, row in df_book_lines.iterrows():\r\n",
        "  new_row = row['line']\r\n",
        "  for word in row['line'].split(' '):\r\n",
        "    if word in list_names:\r\n",
        "      new_row = row['line'].replace(word, '@name')\r\n",
        "    if word in [':','[',']','_']:\r\n",
        "      new_row = row['line'] = '[DELETE]'\r\n",
        "\r\n",
        "  clear_output(wait=True)\r\n",
        "  print(index)   \r\n",
        "  list_new_rows.append(new_row)"
      ],
      "execution_count": 162,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "348759\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o0ibh4v34e_X"
      },
      "source": [
        "df_book_lines.iloc[:,0] = list_new_rows"
      ],
      "execution_count": 163,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "id": "oDPp232x4zEG",
        "outputId": "9dc8c33c-4431-41ba-9800-06c5a7784a94"
      },
      "source": [
        "df_book_lines"
      ],
      "execution_count": 164,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>line</th>\n",
              "      <th>len</th>\n",
              "      <th>book</th>\n",
              "      <th>genre</th>\n",
              "      <th>category</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Bestiario es el primer libro de relatos que @name Cortázar publica con su auténtico nombre .</td>\n",
              "      <td>91</td>\n",
              "      <td>Bestiario_-_Julio_Cortazar</td>\n",
              "      <td>cortazar</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>[DELETE]</td>\n",
              "      <td>93</td>\n",
              "      <td>Bestiario_-_Julio_Cortazar</td>\n",
              "      <td>cortazar</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Sorpresa o incomodidad son , en cada texto , un condimento que se agrega al placer indescriptible de su lectura .</td>\n",
              "      <td>110</td>\n",
              "      <td>Bestiario_-_Julio_Cortazar</td>\n",
              "      <td>cortazar</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>[DELETE]</td>\n",
              "      <td>33</td>\n",
              "      <td>Bestiario_-_Julio_Cortazar</td>\n",
              "      <td>cortazar</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>A @name , que gustaba de mis relatos</td>\n",
              "      <td>34</td>\n",
              "      <td>Bestiario_-_Julio_Cortazar</td>\n",
              "      <td>cortazar</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>348755</th>\n",
              "      <td>@name dirección ya formaba parte del pasado , de otra vida que ya no existía .</td>\n",
              "      <td>74</td>\n",
              "      <td>arles_street_no_44_-_Danielle_Steel</td>\n",
              "      <td>Novelas</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>348756</th>\n",
              "      <td>Ahora le pertenecía a otra familia que la querría tanto como ella .</td>\n",
              "      <td>66</td>\n",
              "      <td>arles_street_no_44_-_Danielle_Steel</td>\n",
              "      <td>Novelas</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>348757</th>\n",
              "      <td>El @number de @name Street era un episodio de su vida , mucho más que una simple casa .</td>\n",
              "      <td>82</td>\n",
              "      <td>arles_street_no_44_-_Danielle_Steel</td>\n",
              "      <td>Novelas</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>348758</th>\n",
              "      <td>Aquel capítulo ya estaba cerrado .</td>\n",
              "      <td>33</td>\n",
              "      <td>arles_street_no_44_-_Danielle_Steel</td>\n",
              "      <td>Novelas</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>348759</th>\n",
              "      <td>El siguiente acababa de empezar .</td>\n",
              "      <td>32</td>\n",
              "      <td>arles_street_no_44_-_Danielle_Steel</td>\n",
              "      <td>Novelas</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>348760 rows × 5 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                                                                                     line  ...  category\n",
              "0                            Bestiario es el primer libro de relatos que @name Cortázar publica con su auténtico nombre .  ...      Book\n",
              "1                                                                                                                [DELETE]  ...      Book\n",
              "2       Sorpresa o incomodidad son , en cada texto , un condimento que se agrega al placer indescriptible de su lectura .  ...      Book\n",
              "3                                                                                                                [DELETE]  ...      Book\n",
              "4                                                                                    A @name , que gustaba de mis relatos  ...      Book\n",
              "...                                                                                                                   ...  ...       ...\n",
              "348755                                     @name dirección ya formaba parte del pasado , de otra vida que ya no existía .  ...      Book\n",
              "348756                                                Ahora le pertenecía a otra familia que la querría tanto como ella .  ...      Book\n",
              "348757                            El @number de @name Street era un episodio de su vida , mucho más que una simple casa .  ...      Book\n",
              "348758                                                                                 Aquel capítulo ya estaba cerrado .  ...      Book\n",
              "348759                                                                                  El siguiente acababa de empezar .  ...      Book\n",
              "\n",
              "[348760 rows x 5 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 164
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZV4sVI-pg758",
        "outputId": "26ce6164-679b-4ba5-f44a-4571b47fa4ae"
      },
      "source": [
        "# Obsrevamos lineas repetidas\r\n",
        "df_book_lines.line.value_counts().head(12)"
      ],
      "execution_count": 165,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[DELETE]                                                                                              10653\n",
              "                                                                                                         51\n",
              "De Canciones de Mihyar el de Damasco @number                                                             21\n",
              "Versión de @name Martínez Montávez                                                                       21\n",
              "A    È  Y  ÍA                                                                                            18\n",
              "ÓN                                                                                                       10\n",
              "Y                                                                                                        10\n",
              "Es una de las autoras más conocidas y leídas en el mundo entero .                                        10\n",
              "De sus novelas , traducidas a veintiocho idiomas , se han vendido @number millones de ejemplares .       10\n",
              "En @number se le otorgó la ciudadanía francesa .                                                          9\n",
              "Í                                                                                                         9\n",
              "Cortázar también vivió en @name , España y Suiza .                                                        9\n",
              "Name: line, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 165
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "id": "8jmx1YJYYrNq",
        "outputId": "c92355b5-4187-4d2c-ff07-243ce7596da0"
      },
      "source": [
        "#Creamos una mascara para el borrado de \r\n",
        "\r\n",
        "borrar = df_book_lines.line.value_counts().head(12).index.tolist()\r\n",
        "mask_del = df_book_lines.line.apply(lambda x: True if x in borrar else False) \r\n",
        "df_book_lines[mask_del]"
      ],
      "execution_count": 166,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>line</th>\n",
              "      <th>len</th>\n",
              "      <th>book</th>\n",
              "      <th>genre</th>\n",
              "      <th>category</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>[DELETE]</td>\n",
              "      <td>93</td>\n",
              "      <td>Bestiario_-_Julio_Cortazar</td>\n",
              "      <td>cortazar</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>[DELETE]</td>\n",
              "      <td>33</td>\n",
              "      <td>Bestiario_-_Julio_Cortazar</td>\n",
              "      <td>cortazar</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>[DELETE]</td>\n",
              "      <td>102</td>\n",
              "      <td>Bestiario_-_Julio_Cortazar</td>\n",
              "      <td>cortazar</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>[DELETE]</td>\n",
              "      <td>121</td>\n",
              "      <td>Bestiario_-_Julio_Cortazar</td>\n",
              "      <td>cortazar</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>91</th>\n",
              "      <td>[DELETE]</td>\n",
              "      <td>104</td>\n",
              "      <td>Bestiario_-_Julio_Cortazar</td>\n",
              "      <td>cortazar</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>347610</th>\n",
              "      <td>[DELETE]</td>\n",
              "      <td>35</td>\n",
              "      <td>arles_street_no_44_-_Danielle_Steel</td>\n",
              "      <td>Novelas</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>348297</th>\n",
              "      <td>[DELETE]</td>\n",
              "      <td>105</td>\n",
              "      <td>arles_street_no_44_-_Danielle_Steel</td>\n",
              "      <td>Novelas</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>348503</th>\n",
              "      <td>[DELETE]</td>\n",
              "      <td>123</td>\n",
              "      <td>arles_street_no_44_-_Danielle_Steel</td>\n",
              "      <td>Novelas</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>348677</th>\n",
              "      <td>[DELETE]</td>\n",
              "      <td>88</td>\n",
              "      <td>arles_street_no_44_-_Danielle_Steel</td>\n",
              "      <td>Novelas</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>348709</th>\n",
              "      <td>[DELETE]</td>\n",
              "      <td>79</td>\n",
              "      <td>arles_street_no_44_-_Danielle_Steel</td>\n",
              "      <td>Novelas</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>10831 rows × 5 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "            line  len                                 book     genre category\n",
              "1       [DELETE]   93           Bestiario_-_Julio_Cortazar  cortazar     Book\n",
              "3       [DELETE]   33           Bestiario_-_Julio_Cortazar  cortazar     Book\n",
              "28      [DELETE]  102           Bestiario_-_Julio_Cortazar  cortazar     Book\n",
              "41      [DELETE]  121           Bestiario_-_Julio_Cortazar  cortazar     Book\n",
              "91      [DELETE]  104           Bestiario_-_Julio_Cortazar  cortazar     Book\n",
              "...          ...  ...                                  ...       ...      ...\n",
              "347610  [DELETE]   35  arles_street_no_44_-_Danielle_Steel   Novelas     Book\n",
              "348297  [DELETE]  105  arles_street_no_44_-_Danielle_Steel   Novelas     Book\n",
              "348503  [DELETE]  123  arles_street_no_44_-_Danielle_Steel   Novelas     Book\n",
              "348677  [DELETE]   88  arles_street_no_44_-_Danielle_Steel   Novelas     Book\n",
              "348709  [DELETE]   79  arles_street_no_44_-_Danielle_Steel   Novelas     Book\n",
              "\n",
              "[10831 rows x 5 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 166
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8prSvPXhXXeq"
      },
      "source": [
        "# Ejecutamos el Drop\r\n",
        "df_book_lines.drop(df_book_lines[mask_del].index, inplace=True)"
      ],
      "execution_count": 167,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cwUNKYRiaQdC",
        "outputId": "6ee471a6-d834-4fec-fa9b-b2d5baa1d7b0"
      },
      "source": [
        "# Evaluamos el shape actual\r\n",
        "df_book_lines.drop_duplicates(inplace=True)\r\n",
        "df_book_lines.shape[0]"
      ],
      "execution_count": 168,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "337358"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 168
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ffCo96XDF2ff"
      },
      "source": [
        "def remove_special_chars(sentence):\r\n",
        "  sentence = sentence.lower().strip()\r\n",
        "  accentedCharacters = \"áéíóúÁÉÍÓÚñÑäëïöüÜ\"\r\n",
        "  sentence = re.sub(r\"[^a-zA-Z\" + accentedCharacters + \"@?¿¡.!,]+\", \" \", sentence)\r\n",
        "  sentence = re.sub(r\"([?¿¡.!,])\", r\" \\1 \", sentence)\r\n",
        "  sentence = sentence.strip()\r\n",
        "  sentence = sentence.strip('.')\r\n",
        "  sentence = sentence.strip(',')\r\n",
        "  return sentence"
      ],
      "execution_count": 212,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JGMH8vRQF2fg"
      },
      "source": [
        "df_book_lines.iloc[:,0] = df_book_lines.iloc[:,0].apply(remove_special_chars)"
      ],
      "execution_count": 213,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e4GcMa6TaaER"
      },
      "source": [
        "# Regeneramos la longitud en caracteres post procesamiento\r\n",
        "df_book_lines.len = df_book_lines.line.apply(lambda x : len(x))"
      ],
      "execution_count": 202,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qmLAjBZmfUjH",
        "outputId": "5d3485fd-dbce-4f04-f494-3092ba581c3e"
      },
      "source": [
        "# Creamos una mascara para eliminar lineas con menos de 35 caracteres\r\n",
        "\r\n",
        "mask_largo = df_book_lines.len < 35\r\n",
        "print(mask_largo.sum())"
      ],
      "execution_count": 203,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "893\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Rb5Tuo_UfxNr",
        "outputId": "d5c3a261-fbab-4215-d510-d015657bbfce"
      },
      "source": [
        "# Ejecutamos el Drop\r\n",
        "# Evaluamos el shape actual\r\n",
        "df_book_lines.drop(df_book_lines[mask_largo].index, inplace=True)\r\n",
        "df_book_lines.shape[0]"
      ],
      "execution_count": 204,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "316618"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 204
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 514
        },
        "id": "nvZLIGCof6di",
        "outputId": "a482c201-4fc3-4bfe-efd9-19d985a41147"
      },
      "source": [
        "# Obtenemos algunos samples para evaluar el resultado de nuestro corpus\r\n",
        "\r\n",
        "df_book_lines.sample(15)"
      ],
      "execution_count": 214,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>line</th>\n",
              "      <th>len</th>\n",
              "      <th>book</th>\n",
              "      <th>genre</th>\n",
              "      <th>category</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>74822</th>\n",
              "      <td>es la mano que toma todo lo que le pertenece en mi cuerpo</td>\n",
              "      <td>59</td>\n",
              "      <td>_frente_al_espejo_-_Denis_Hernandez</td>\n",
              "      <td>poesía</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>345043</th>\n",
              "      <td>el pequeño asintió  ,  así que @name llenó un bol de cereales  ,  los cubrió de leche y le dio un plato para el plátano</td>\n",
              "      <td>121</td>\n",
              "      <td>arles_street_no_44_-_Danielle_Steel</td>\n",
              "      <td>Novelas</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>155763</th>\n",
              "      <td>gordon querrá que vuelva a parís en cuanto me recupere</td>\n",
              "      <td>56</td>\n",
              "      <td>El_beso_-_Danielle_Steel</td>\n",
              "      <td>Novelas</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>88593</th>\n",
              "      <td>en tibias caras de espantados ojos</td>\n",
              "      <td>36</td>\n",
              "      <td>_grandes_mujeres_-_Alfonsina_Storni</td>\n",
              "      <td>poesía</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34751</th>\n",
              "      <td>mamá no ganaba para sustos  ,  entre el chico de la modista que se había quemado el brazo con leche hirviendo y el boby sarnoso</td>\n",
              "      <td>129</td>\n",
              "      <td>Las-armas-secretas-Julio-Cortazar</td>\n",
              "      <td>cortazar</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50014</th>\n",
              "      <td>tener opiniones es inevitable  ,  normal tener convicciones lo es menos</td>\n",
              "      <td>73</td>\n",
              "      <td>ente_de_haber_nacido_-_E._M._Cioran</td>\n",
              "      <td>filosofia</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>187592</th>\n",
              "      <td>para aquellos que jamás conocieron esas cosas  ,  esto tiene el poder de una fuerza hipnótica</td>\n",
              "      <td>95</td>\n",
              "      <td>El_anillo_-_Danielle_Steel</td>\n",
              "      <td>Novelas</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>159762</th>\n",
              "      <td>ahora  ,  mis padres ya no serán un problema</td>\n",
              "      <td>46</td>\n",
              "      <td>Guarida_de_condenados_-_A._Rolcest</td>\n",
              "      <td>Novelas</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>283129</th>\n",
              "      <td>pero un cruel golpe del destino había acabado con todo</td>\n",
              "      <td>56</td>\n",
              "      <td>El_rancho_-_Danielle_Steel</td>\n",
              "      <td>Novelas</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27508</th>\n",
              "      <td>@number h  .  @number c al sol  ,  lo que no nos ha impedido explorar  ,  como intrépidos expedicionarios que somos  ,  el arqueódromo</td>\n",
              "      <td>136</td>\n",
              "      <td>Los-autonautas-de-la-cosmopista-Julio-Cortazar</td>\n",
              "      <td>cortazar</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>68348</th>\n",
              "      <td>@name belleza no admite explicación racional se cree en ella y punto</td>\n",
              "      <td>70</td>\n",
              "      <td>Vetas_profundas_-_Fernando_Aramburu</td>\n",
              "      <td>poesía</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25753</th>\n",
              "      <td>me refiero a partir de la escena @number porque en todo lo anterior ya has visto que estamos completamente de acuerdo</td>\n",
              "      <td>119</td>\n",
              "      <td>Cartas_1955-1964_-_Julio_Cortazar</td>\n",
              "      <td>cortazar</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>101888</th>\n",
              "      <td>@name primera sonrisa desde que dejó caer la bomba en el estudio</td>\n",
              "      <td>66</td>\n",
              "      <td>a_extrana_propuesta_-_Allison_Leigh</td>\n",
              "      <td>Novelas</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>201766</th>\n",
              "      <td>todas las conversaciones sobre el tema habían finalizado bruscamente meses antes</td>\n",
              "      <td>82</td>\n",
              "      <td>Aguila_solitaria_-_Danielle_Steel</td>\n",
              "      <td>Novelas</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>273400</th>\n",
              "      <td>pero ahora que ha alcanzado la mayoría de edad  ,  tengo que informarle</td>\n",
              "      <td>73</td>\n",
              "      <td>Acto_de_fe_-_Danielle_Steel</td>\n",
              "      <td>Novelas</td>\n",
              "      <td>Book</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                                                                                                          line  ...  category\n",
              "74822                                                                                es la mano que toma todo lo que le pertenece en mi cuerpo  ...      Book\n",
              "345043                 el pequeño asintió  ,  así que @name llenó un bol de cereales  ,  los cubrió de leche y le dio un plato para el plátano  ...      Book\n",
              "155763                                                                                  gordon querrá que vuelva a parís en cuanto me recupere  ...      Book\n",
              "88593                                                                                                       en tibias caras de espantados ojos  ...      Book\n",
              "34751          mamá no ganaba para sustos  ,  entre el chico de la modista que se había quemado el brazo con leche hirviendo y el boby sarnoso  ...      Book\n",
              "50014                                                                  tener opiniones es inevitable  ,  normal tener convicciones lo es menos  ...      Book\n",
              "187592                                           para aquellos que jamás conocieron esas cosas  ,  esto tiene el poder de una fuerza hipnótica  ...      Book\n",
              "159762                                                                                            ahora  ,  mis padres ya no serán un problema  ...      Book\n",
              "283129                                                                                  pero un cruel golpe del destino había acabado con todo  ...      Book\n",
              "27508   @number h  .  @number c al sol  ,  lo que no nos ha impedido explorar  ,  como intrépidos expedicionarios que somos  ,  el arqueódromo  ...      Book\n",
              "68348                                                                     @name belleza no admite explicación racional se cree en ella y punto  ...      Book\n",
              "25753                    me refiero a partir de la escena @number porque en todo lo anterior ya has visto que estamos completamente de acuerdo  ...      Book\n",
              "101888                                                                        @name primera sonrisa desde que dejó caer la bomba en el estudio  ...      Book\n",
              "201766                                                        todas las conversaciones sobre el tema habían finalizado bruscamente meses antes  ...      Book\n",
              "273400                                                                 pero ahora que ha alcanzado la mayoría de edad  ,  tengo que informarle  ...      Book\n",
              "\n",
              "[15 rows x 5 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 214
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H2rvUUwKH4OY"
      },
      "source": [
        "# Guardamos el resultado\r\n",
        "df_book_lines.to_csv('/content/drive/My Drive/Colab Notebooks/desafio final/train_corpus_v2.csv')"
      ],
      "execution_count": 215,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fFJ4YY7u1n7u"
      },
      "source": [
        "# Generamos una funcion para buscar palabras extranjeras, nos sirve para evaluar pusibles errores\r\n",
        "def find_word(palabra):\r\n",
        "  for index, row in df_book_lines.iterrows():\r\n",
        "    new_row = row['line']\r\n",
        "    for word in row['line'].split(' '):\r\n",
        "      if word in [palabra]:\r\n",
        "        print(row['book'])\r\n",
        "  "
      ],
      "execution_count": 148,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WrUBoA81uthH",
        "outputId": "7e9e6940-d719-4be9-d4e5-28891957558f"
      },
      "source": [
        "find_word('passagem')"
      ],
      "execution_count": 149,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Poemas_-_Fernando_Pessoa\n",
            "Poemas_-_Fernando_Pessoa\n",
            "Poemas_-_Fernando_Pessoa\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hKeqAVlF5bS-"
      },
      "source": [
        "# mask_clean_book = df_book_lines.book == 'Poemas_-_Fernando_Pessoa'\r\n",
        "# mask_clean_book.sum()\r\n",
        "\r\n",
        "# print(df_book_lines.shape[0])\r\n",
        "# df_book_lines.drop(df_book_lines[mask_clean_book].index, inplace=True)\r\n",
        "# print(df_book_lines.shape[0])"
      ],
      "execution_count": 92,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sXzSwG5k0F2x"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}