Add_new_books_dataset.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "mount_file_id": "1RkffNWFgkvVFr5kFF7ACYyNQFLWb8PwR",
      "authorship_tag": "ABX9TyMxINPv8oQ+MP5CV9/QpSbK"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 34,
      "metadata": {
        "id": "jwrSXR4nFqKJ"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import json\n",
        "import requests\n",
        "from bs4 import BeautifulSoup as bs\n",
        "from IPython.display import clear_output, display"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "### ALL FUNCTIONS HERE ###"
      ],
      "metadata": {
        "id": "FrN1LbHXGGor"
      },
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def getBookMeta(isbn):\n",
        "    api_url = 'https://www.googleapis.com/books/v1/volumes?q=isbn:'\n",
        "    isbn_number = isbn\n",
        "    url = api_url + str(isbn_number)\n",
        "    return json.loads(requests.get(url).content)"
      ],
      "metadata": {
        "id": "F72zrjd7GNVX"
      },
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "apiRestBooks = []\n",
        "def apiRest(isbn):\n",
        "  apiRestBooks.append(isbn)"
      ],
      "metadata": {
        "id": "qqCFaGCSGPZW"
      },
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "## check is the book with ISBN has data on Goodreads return True/False ##\n",
        "def GRhasBook(isbn):\n",
        "  url = 'https://www.goodreads.com/book/review_counts.json?isbns='\n",
        "  result = requests.get(url+str(isbn)).text\n",
        "  if result == 'No books match those ISBNs.':\n",
        "    return False\n",
        "  elif result == 'No ISBNs specified.':\n",
        "    return False\n",
        "  else:\n",
        "    return True"
      ],
      "metadata": {
        "id": "2fzYSjHHGYem"
      },
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def getGRinit(isbn):\n",
        "  if GRhasBook(isbn):\n",
        "    url = 'https://www.goodreads.com/book/review_counts.json?isbns='\n",
        "    result = json.loads(requests.get(url+str(isbn)).content)\n",
        "    return result.get('books')\n",
        "  else:\n",
        "    print('NO BOOKS FOUND ... ', isbn)"
      ],
      "metadata": {
        "id": "UVq1IxVxGa3x"
      },
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def GRhasRatings(isbn):\n",
        "  stat = getGRreviewStat(isbn)[0]\n",
        "  ratings_cnt = stat[2]\n",
        "  reviews_cnt = stat[3]\n",
        "  if ratings_cnt == 0 or reviews_cnt == 0:\n",
        "    return False\n",
        "  else:\n",
        "    return True"
      ],
      "metadata": {
        "id": "L9CQazbDGeid"
      },
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "## return list of reviews stat of the book ##\n",
        "def getGRreviewStat(isbn):\n",
        "  bookStats = []\n",
        "  url = 'https://www.goodreads.com/book/review_counts.json?isbns='\n",
        "  if GRhasBook(isbn):\n",
        "    meta = json.loads(requests.get(url+str(isbn)).text).get('books')[0]\n",
        "    ratings_cnt = meta.get('work_ratings_count')\n",
        "    text_reviews_cnt = meta.get('work_text_reviews_count')\n",
        "    reviews_cnt = meta.get('work_reviews_count')\n",
        "    avg_rating = meta.get('average_rating')\n",
        "    bookStats.append([avg_rating, reviews_cnt, ratings_cnt, text_reviews_cnt,])\n",
        "    return bookStats\n",
        "  else:\n",
        "    return [['','','','','']]"
      ],
      "metadata": {
        "id": "VI9mDA4wGhT-"
      },
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "noBooksList = []\n",
        "noStatList = []\n",
        "## return list of book attributes ##\n",
        "def getGRmeta(isbn):\n",
        "  if GRhasBook(isbn):\n",
        "    url = 'https://www.goodreads.com/book/isbn/'+str(isbn)\n",
        "    page = requests.get(url)\n",
        "    global soup_author\n",
        "    soup_author = bs(page.content, 'html.parser')\n",
        "\n",
        "    #if not not404:\n",
        "    #  return\n",
        "\n",
        "    ### /!\\ ATTENTION /!\\ ###\n",
        "    ### loop for author name due to some kind of weird url request or dynamic content thing can lead to non auther =) ###\n",
        "    ### take so long time but can get complete data used when GoogleBooks API not found the books ONLY ###\n",
        "    while len(soup_author.find_all('a', class_ = 'authorName')) < 1:\n",
        "      url = 'https://www.goodreads.com/book/isbn/'+str(isbn)\n",
        "      page = requests.get(url)\n",
        "      soup_author = bs(page.content, 'html.parser')\n",
        "\n",
        "    bookAttributes = []\n",
        "    author_list = []\n",
        "    for author in soup_author.find_all('a', class_ = 'authorName'):\n",
        "      author_list.append(author.string)\n",
        "    bookTitle = soup_author.find_all('title')[0].string\n",
        "\n",
        "    ### book statistics process ###\n",
        "    bookStat = getGRreviewStat(isbn)[0] # list of book stat [[average_rating, #reviews, #ratings, #text_reviews]]\n",
        "    rating_avg = bookStat[0]\n",
        "    reviews_cnt = bookStat[1]\n",
        "    ratings_cnt = bookStat[2]\n",
        "    text_reviews_cnt = bookStat[3]\n",
        "    ### end of statistic process ###\n",
        "\n",
        "    ### Append all attributes ###\n",
        "    bookAttributes.append([author_list, bookTitle])\n",
        "    return bookAttributes\n",
        "  else:\n",
        "    print('no book: ', isbn)"
      ],
      "metadata": {
        "id": "TFdfnyseGoXA"
      },
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def getID():\n",
        "  id = len(dfBooks) + 1\n",
        "  return id"
      ],
      "metadata": {
        "id": "jmTSF-3rGtDg"
      },
      "execution_count": 43,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def addRow(meta):\n",
        "    adding_book = pd.DataFrame(meta, columns=['id', 'isbn_10', 'isbn_13', 'isbn_other', 'isbn_book', 'authors', 'title', 'subtitle', 'publisher', 'published_date', 'page_count', 'categories', 'language', 'google_desc', 'rating_avg', '#reviews', '#ratings', '#text_reviews', 'thumbnail', 'genre', 'goodreads_desc', 'text_reviews', 'also_enjoy', 'status'])\n",
        "    global dfBooks\n",
        "    dfBooks = pd.concat([dfBooks, adding_book], ignore_index = True)"
      ],
      "metadata": {
        "id": "ogJJesdrGvAn"
      },
      "execution_count": 44,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def addBook(isbn):\n",
        "    book_meta = getBookMeta(isbn)\n",
        "    total_items = book_meta['totalItems']\n",
        "    if total_items == 0:\n",
        "      global noBooksNum\n",
        "      noBooksList.append(isbn)\n",
        "      #noBooksNum = noBooksNum + 1\n",
        "      clear_output(wait=True)\n",
        "      display(len(noBooksList),' no book found: ', isbn)\n",
        "      return\n",
        "\n",
        "    book_meta_item = book_meta.get('items')\n",
        "    meta = []\n",
        "\n",
        "    ## attributes ##\n",
        "    #try:\n",
        "    for item in range(len(book_meta_item)):\n",
        "      metaData = book_meta_item[item].get('volumeInfo')\n",
        "      #google_id = book_meta_item[item]['id']\n",
        "\n",
        "      ### isbn process ###\n",
        "      if GRhasBook(isbn):\n",
        "        GRini = getGRinit(isbn)[0]\n",
        "        # initialize isbn #\n",
        "        isbn_10 = GRini.get('isbn')\n",
        "        isbn_13 = str(GRini.get('isbn13'))\n",
        "        isbn_other = None\n",
        "        isbn_book = isbn\n",
        "      else:\n",
        "        isbn_10 = None\n",
        "        isbn_13 = None\n",
        "        isbn_other = None\n",
        "        isbn_book = isbn\n",
        "      ### end of isbn process ###\n",
        "\n",
        "      authors = metaData.get('authors','')\n",
        "      title = metaData.get('title','')\n",
        "      subtitle = metaData.get('subtitle','')\n",
        "      publisher = metaData.get('publisher','')\n",
        "      published_date = metaData.get('publishedDate','')\n",
        "      page_count = metaData.get('pageCount','')\n",
        "      categories = metaData.get('categories','')\n",
        "      language = metaData.get('language','')\n",
        "      google_desc = metaData.get('description','')\n",
        "      #rating_avg = metaData.get('averageRating','')\n",
        "      #rating_count = metaData.get('ratingsCount','')\n",
        "\n",
        "      ### thumbnail process ###\n",
        "      global img_url\n",
        "      if metaData.get('imageLinks','') is '':\n",
        "        img_url = ''\n",
        "      else:\n",
        "        img_url = metaData.get('imageLinks','').get('thumbnail','')\n",
        "      ### end thumbnail process ###\n",
        "\n",
        "\n",
        "      ### check if the book has ISBN_13 (ONLY ISBN_13 WILL GET STAT) ###\n",
        "      if isbn_13 is None:\n",
        "        noStatList.append(isbn)\n",
        "        #global books_cnt\n",
        "        #books_cnt = books_cnt + 1\n",
        "        global dfBooks\n",
        "        id = getID()\n",
        "        meta.append([id, isbn_10, isbn_13, isbn_other, isbn_book, authors, title, subtitle, publisher, published_date, page_count, categories, language, google_desc, None, None, None, None, img_url, None, None, None, None, -1])\n",
        "        addRow(meta)\n",
        "        meta = []\n",
        "        continue\n",
        "\n",
        "      ### book statistics process ###\n",
        "      bookStat = getGRreviewStat(isbn_13)[0] # list of book stat [[average_rating, #reviews, #ratings, #text_reviews]]\n",
        "      rating_avg = bookStat[0]\n",
        "      reviews_cnt = bookStat[1]\n",
        "      ratings_cnt = bookStat[2]\n",
        "      text_reviews_cnt = bookStat[3]\n",
        "      ### end of statistic process ###\n",
        "\n",
        "      #global books_cnt\n",
        "      #books_cnt = books_cnt + 1\n",
        "      id = getID()\n",
        "      meta.append([id, isbn_10, isbn_13, isbn_other, isbn_book, authors, title, subtitle, publisher, published_date, page_count, categories, language, google_desc, rating_avg, reviews_cnt, ratings_cnt, text_reviews_cnt, img_url, None, None, None, None, -1])\n",
        "\n",
        "      addRow(meta)\n",
        "      meta = []\n",
        "      \n",
        "    #adding_book = pd.DataFrame(meta, columns=['id', 'isbn_10', 'isbn_13', 'isbn_other', 'isbn_book', 'authors', 'title', 'subtitle', 'publisher', 'published_date', 'page_count', 'categories', 'language', 'google_desc', 'rating_avg', '#reviews', '#ratings', '#text_reviews', 'thumbnail'])\n",
        "    #global dfBooks\n",
        "    #dfBooks = pd.concat([dfBooks, adding_book], ignore_index = True)\n"
      ],
      "metadata": {
        "id": "cBcTOss3GwaX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "17a892dd-8cfe-430a-c343-6e7bce1ebc1a"
      },
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<>:50: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
            "<>:50: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
            "<ipython-input-45-ca139abc6c77>:50: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
            "  if metaData.get('imageLinks','') is '':\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "jsonError = []\n",
        "def newBooks(books_isbn):\n",
        "  for index in range(len(books_isbn)):\n",
        "    isbn = books_isbn.iloc[index]['Barcode']\n",
        "    if isDuplicate(isbn):\n",
        "      print('Aready has book: ' + isbn)\n",
        "      continue\n",
        "    try:\n",
        "      ### PROGRESS ###\n",
        "      clear_output(wait=True)\n",
        "      print('ADDING ... ',isbn , '[', index,'/',len(books_isbn),']')\n",
        "      print('NO BOOKS FOUND: ', len(noBooksList), 'BOOKS')\n",
        "      print('API OUT: ', len(apiRestBooks))\n",
        "      print('JSON ERROR: ', len(jsonError))\n",
        "      ################\n",
        "      addBook(isbn)\n",
        "    except KeyError as e: # key error\n",
        "      if e.args[0] == 'totalItems':\n",
        "        apiRestNum = apiRestNum + 1\n",
        "        #clear_output(wait=True)\n",
        "        display(apiRestNum, ' API OUT')\n",
        "        apiRest(isbn)\n",
        "    #except ValueError as v: # JSON ERROR\n",
        "    #  jsonError.append(isbn)\n",
        "    #  dfErr = {'Barcode':isbn}\n",
        "    #  books_isbn = books_isbn.append(dfErr, ignore_index=True)\n",
        "    #  display([len(jsonError),' JSON ERROR'])"
      ],
      "metadata": {
        "id": "-Jl6wsAvG09i"
      },
      "execution_count": 46,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def retryAPIout(apiRestBooks):\n",
        "  tempBooks = apiRestBooks\n",
        "  while len(apiRestBooks) != 0:\n",
        "    for isbn in apiRestBooks:\n",
        "      try:\n",
        "        addBook(isbn)\n",
        "        apiRestBooks.pop(apiRestBooks.index(isbn))\n",
        "        apiRestNum = apiRestNum -1\n",
        "        clear_output(wait=True)\n",
        "        print(apiRestNum, ' LEFT')\n",
        "      except KeyError as e:\n",
        "        if e.args[0] == 'totalItems':\n",
        "          continue\n",
        "          #clear_output(wait=True)\n",
        "          #display('API OUT')\n",
        "      except ValueError as v: # JSON ERROR\n",
        "        jsonError.append(isbn)\n",
        "        dfErr = {'Barcode':isbn}\n",
        "        books_isbn = books_isbn.append(dfErr, ignore_index=True)\n",
        "        display([len(jsonError),' JSON ERROR'])"
      ],
      "metadata": {
        "id": "4IluQ35zG9rU"
      },
      "execution_count": 47,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "list404 = []\n",
        "list500 = []\n",
        "listOther = []\n",
        "\n",
        "def not404(isbn):\n",
        "  url = 'https://www.goodreads.com/book/isbn/'+str(isbn)\n",
        "  page = requests.get(url)\n",
        "  if page.status_code == 200:\n",
        "    return True\n",
        "  elif page.status_code == 404:\n",
        "    list404.append(isbn)\n",
        "    display('404')\n",
        "    return False\n",
        "  elif page.status_code == 500:\n",
        "    list500.append(isbn)\n",
        "    display('500')\n",
        "    return False\n",
        "  else:\n",
        "    listOther.append(isbn)\n",
        "    display('SOME HTTP ERROR')\n",
        "    return False"
      ],
      "metadata": {
        "id": "O-c_6RiCHOZW"
      },
      "execution_count": 48,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def getContent(isbn):\n",
        "  #clear_output(wait=True)\n",
        "  print('GETTING CONTENT >>> ', isbn)\n",
        "  if GRhasBook(isbn) and not404(isbn):\n",
        "    url= 'https://www.goodreads.com/book/isbn/' + str(isbn)\n",
        "    page = requests.get(url) \n",
        "    soup = bs(page.content, 'html.parser')\n",
        "    while checkContent(soup, isbn):\n",
        "      page = requests.get(url) \n",
        "      soup = bs(page.content, 'html.parser')\n",
        "    return soup\n",
        "  else:\n",
        "    display('ISBN NOT FOUND OR HTTP ERROR')"
      ],
      "metadata": {
        "id": "7psZuUm3HS0s"
      },
      "execution_count": 49,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def isGenre(isbn):\n",
        "  acceptance_rounds = 4\n",
        "  while acceptance_rounds > 0:\n",
        "    url = 'https://www.goodreads.com/book/isbn/' + str(isbn)\n",
        "    page = requests.get(url)\n",
        "    soup = bs(page.content, 'html.parser')\n",
        "    genre = soup.find_all('a', class_ = 'actionLinkLite bookPageGenreLink')\n",
        "    #clear_output(wait=True)\n",
        "    if len(genre) > 0:\n",
        "      display('GENRE FOUND')\n",
        "      return False\n",
        "      break\n",
        "    else:\n",
        "      acceptance_rounds = acceptance_rounds - 1\n",
        "      print('NO GENRE ACCEPTANCE ROUNDS: ', acceptance_rounds)\n",
        "  return True"
      ],
      "metadata": {
        "id": "Vi1WJ3yWHU2J"
      },
      "execution_count": 50,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def checkContent(content, isbn):\n",
        "  if content is None: ## content is None will lead to AttributeError ## use 'is' instead of '=='\n",
        "    return True\n",
        "  ### check genre ###\n",
        "  genre = content.find_all('a', class_ = 'actionLinkLite bookPageGenreLink')\n",
        "  ### check text reviews ###\n",
        "  textReviews = content.find_all('span', class_ = 'readable')\n",
        "\n",
        "  ### CHECK CONDITION ###\n",
        "  if len(textReviews) == 0: ## no text reviews get new request(true)\n",
        "    return True\n",
        "  elif len(genre) != 0 and len(textReviews) != 0: ## Good conent\n",
        "    return False\n",
        "  elif isGenre(isbn) and len(textReviews) != 0: ## real content have no genre and have text -> good content no need to get new request\n",
        "    return False\n",
        "  else: ## real content have genre but the content we get not found -> get new request(true)\n",
        "    return True"
      ],
      "metadata": {
        "id": "efs1B405HYKf"
      },
      "execution_count": 51,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def getGenre(content, isbn):\n",
        "  genre_list = []\n",
        "  if content is None:\n",
        "    content = getContent(isbn)\n",
        "  for index in range(len(content.find_all('a', class_ = 'actionLinkLite bookPageGenreLink'))):\n",
        "    genre = content.find_all('a', class_ = 'actionLinkLite bookPageGenreLink')[index].string\n",
        "    genre_list.append(genre)\n",
        "  return genre_list"
      ],
      "metadata": {
        "id": "1hDpzfaUHbQl"
      },
      "execution_count": 52,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "################################    #\n",
        "### ADD BOOKS FROM GOODREADS ### #######\n",
        "################################    #\n",
        "## new initialize\n",
        "def addGRbooks(noBooksList):\n",
        "  noBooksNum = len(noBooksList)\n",
        "  for index in range(len(noBooksList)):\n",
        "    isbn = noBooksList[index]\n",
        "    clear_output(wait=True)\n",
        "    display(['CHECKING >>> ', isbn])\n",
        "    display(['NO INFO BOOKS: ', noBooksNum, ' LEFT'])\n",
        "    display(['404 REQUEST: ', len(list404)])\n",
        "    if GRhasBook(isbn) and not404(isbn):\n",
        "      GRini = getGRinit(isbn)[0]\n",
        "      meta = getGRmeta(isbn)[0]\n",
        "      ### ISBN ###\n",
        "      isbn_10 = GRini.get('isbn')\n",
        "      isbn_13 = str(GRini.get('isbn13'))\n",
        "      ### book statistics process ###\n",
        "      bookStat = getGRreviewStat(isbn)[0] # list of book stat [[average_rating, #reviews, #ratings, #text_reviews]]\n",
        "      rating_avg = bookStat[0]\n",
        "      reviews_cnt = bookStat[1]\n",
        "      ratings_cnt = bookStat[2]\n",
        "      text_reviews_cnt = bookStat[3]\n",
        "      ### end of statistic process ###\n",
        "      id = getID()\n",
        "      attr = [[id, isbn_10, isbn_13, None, isbn, meta[0], meta[1], None, None, None, None, None, None, None, rating_avg, reviews_cnt, ratings_cnt, text_reviews_cnt, None]]\n",
        "      adding_book = pd.DataFrame(attr, columns=['id', 'isbn_10', 'isbn_13', 'isbn_other', 'isbn_book', 'authors', 'title', 'subtitle', 'publisher', 'published_date', 'page_count', 'categories', 'language', 'google_desc', 'rating_avg', '#reviews', '#ratings', '#text_reviews', 'thumbnail'])\n",
        "      global dfBooks\n",
        "      dfBooks = pd.concat([dfBooks, adding_book], ignore_index = True)\n",
        "      noBooksNum = noBooksNum - 1"
      ],
      "metadata": {
        "id": "9XRfXXswHgbB"
      },
      "execution_count": 53,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def getStars(content, reviews_index):\n",
        "  title = content.find_all('span', class_ = ' staticStars notranslate')[reviews_index].get('title')\n",
        "  stars = 0\n",
        "  if title == 'it was amazing':\n",
        "    stars = 5.0\n",
        "  elif title == 'really liked it':\n",
        "    stars = 4.0\n",
        "  elif title == 'liked it':\n",
        "    stars = 3.0\n",
        "  elif title == 'it was ok':\n",
        "    stars = 2.0\n",
        "  else:\n",
        "    stars = 1.0\n",
        "  return stars"
      ],
      "metadata": {
        "id": "BL1BkW6pIBfp"
      },
      "execution_count": 54,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def getTextReviews(content, isbn):\n",
        "  textReviews_list = []\n",
        "  index = 0\n",
        "  if content is None:\n",
        "    content = getContent(isbn)\n",
        "  cnt = len(content.find_all('span', class_ = ' staticStars notranslate'))\n",
        "  #clear_output(wait=True)\n",
        "  print('GETTING TEXT REVIEWS >>> ', cnt, ' REVIEWS')\n",
        "  for line in content.find_all('span', class_ = 'readable'):\n",
        "    if cnt <= 0:\n",
        "      break \n",
        "    text = line.text.splitlines()\n",
        "    ### concat text reviews ###\n",
        "    fullText = ''\n",
        "    for each in text:\n",
        "      fullText = fullText + each\n",
        "    #display('GETTING STARS >>>')\n",
        "    textReviews_list.append([getStars(content, index), fullText])\n",
        "    index = index + 1\n",
        "    cnt = cnt - 1\n",
        "  return textReviews_list"
      ],
      "metadata": {
        "id": "NDG5qVA6IDor"
      },
      "execution_count": 55,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def getGRdesc(content, isbn):\n",
        "  print('GETTING DESCRIPTION ...')\n",
        "  acct = 2\n",
        "  cont = content.find_all('div', id = 'description')\n",
        "  while len(cont) == 0 and acct != 0:\n",
        "    content = getContent(isbn)\n",
        "    cont = content.find_all('div', id = 'description')\n",
        "    acct = acct - 1\n",
        "  if len(cont) != 0:\n",
        "    return cont[0].text\n",
        "  else:\n",
        "    return None"
      ],
      "metadata": {
        "id": "njtzHXVbIFg_"
      },
      "execution_count": 56,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def getGRthumbnail(content, isbn):\n",
        "  print('GETTING THUMBNAIL ...')\n",
        "  acct = 2\n",
        "  cont = content.find_all('img', id = 'coverImage')\n",
        "  while len(cont) == 0 and acct != 0:\n",
        "    content = getContent(isbn)\n",
        "    cont = content.find_all('img', id = 'coverImage')\n",
        "    acct = acct - 1\n",
        "  if len(cont) != 0:\n",
        "    return cont[0].get('src')\n",
        "  else:\n",
        "    return None"
      ],
      "metadata": {
        "id": "ZQ3W-S5OIHRT"
      },
      "execution_count": 57,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def getGRlang(content, isbn):\n",
        "  print('GETTING LANGUAGE ...')\n",
        "  acct = 2\n",
        "  cont = content.find_all('div', itemprop = 'inLanguage')\n",
        "  while len(cont) == 0 and acct != 0:\n",
        "    content = getContent(isbn)\n",
        "    cont = content.find_all('div', itemprop = 'inLanguage')\n",
        "    acct = acct - 1\n",
        "  if len(cont) != 0:\n",
        "    return cont[0].text\n",
        "  else:\n",
        "    return None"
      ],
      "metadata": {
        "id": "EcmroUtqII2j"
      },
      "execution_count": 58,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def getAlsoEnjoy(content, isbn):\n",
        "  print('GETTING ALSO ENJOYED BOOKS ...')\n",
        "  acct = 2\n",
        "  cont = content.find_all('li', class_ = 'cover')\n",
        "  alsoEnjoy_list = []\n",
        "  while len(cont) == 0 and acct != 0:\n",
        "    content = getContent(isbn)\n",
        "    cont = content.find_all('li', class_ = 'cover')\n",
        "    acct = acct - 1\n",
        "  if len(cont) != 0:\n",
        "    for book in cont:\n",
        "      title = book.find('img').get('alt')\n",
        "      thumbnail = book.find('img').get('src')\n",
        "      alsoEnjoy_list.append([title, thumbnail])\n",
        "    return alsoEnjoy_list\n",
        "  else:\n",
        "    return None"
      ],
      "metadata": {
        "id": "nWSKgYAYIKhA"
      },
      "execution_count": 59,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def getMoreData(dfBooks):\n",
        "  current_batch_data = dfBooks.loc[dfBooks.isbn_13.notna()]\n",
        "  current_batch_data = current_batch_data.loc[current_batch_data.status == -1]\n",
        "\n",
        "  for id in current_batch_data.id:\n",
        "    global current_n\n",
        "    current_n = current_n + 1\n",
        "    try:\n",
        "      isbn = dfBooks.isbn_13[dfBooks.id == id].tolist()[0]\n",
        "      index = dfBooks.isbn_13[dfBooks.id == id].index.tolist()[0]\n",
        "      ### PROGRESS ###\n",
        "      clear_output(wait=True)\n",
        "      global batch_title\n",
        "      print('dfBooks at batch ... ', batch_title)\n",
        "      print('CURRENTLY ', '[', current_n,'/', save_every_n, ']')\n",
        "      print('NOW AT INDEX ... ',index,'/',len(dfBooks))\n",
        "      print(len(jsonError),' JSON ERROR')\n",
        "      ################\n",
        "      if not404(isbn) and GRhasBook(isbn) and GRhasRatings(isbn):\n",
        "        content = getContent(isbn)\n",
        "        genre = getGenre(content, isbn)\n",
        "        text_reviews = getTextReviews(content, isbn)\n",
        "\n",
        "        if content is None:\n",
        "          content = getContent(isbn)\n",
        "\n",
        "        desc = getGRdesc(content, isbn)\n",
        "        alsoEnjoy = getAlsoEnjoy(content, isbn)\n",
        "        thumbnail = getGRthumbnail(content, isbn)\n",
        "\n",
        "        if dfBooks.at[index, 'language'] is None or dfBooks.at[index, 'language'] is np.NaN:\n",
        "          lang = getGRlang(content, isbn)\n",
        "          dfBooks.at[index, 'language'] = lang\n",
        "\n",
        "        dfBooks.at[index, 'genre'] = genre\n",
        "        dfBooks.at[index, 'text_reviews'] = text_reviews\n",
        "        dfBooks.at[index, 'goodreads_desc'] = desc\n",
        "        dfBooks.at[index, 'also_enjoy'] = alsoEnjoy\n",
        "        dfBooks.at[index, 'status'] = 1\n",
        "\n",
        "      else:\n",
        "        dfBooks.at[index, 'status'] = 0\n",
        "\n",
        "    except KeyError as v: # JSON ERROR\n",
        "      print('WHAT')\n",
        "      jsonError.append(isbn)\n",
        "      dfErr = {'Barcode':isbn}\n",
        "      books_isbn = books_isbn.append(dfErr, ignore_index=True)\n",
        "    if current_n == save_every_n:\n",
        "      batch_title = batch_title + 1\n",
        "      path = '/content/drive/MyDrive/Chaloklum bookshop dataset/v_' + dataset_version + '/books_dataset_w_review_' + dataset_version + '_batch_' + str(batch_title) + file_type\n",
        "      dfBooks.to_csv(path, index=False)\n",
        "\n",
        "      dfJSONerror = pd.DataFrame(jsonError)\n",
        "      dfJSONerror.to_csv('/content/drive/MyDrive/Chaloklum bookshop dataset/v_' + dataset_version + '/json_error_' + dataset_version + file_type, index=False)\n",
        "\n",
        "      current_n = 0\n"
      ],
      "metadata": {
        "id": "N-8b2oK0IZzD"
      },
      "execution_count": 60,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def isDuplicate(isbn):\n",
        "  if isbn in dfBooks['isbn_book'].values:\n",
        "    return True\n",
        "  else:\n",
        "    return False"
      ],
      "metadata": {
        "id": "toxz7jEOrL4b"
      },
      "execution_count": 61,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "### END OF FUNCTIONS ###"
      ],
      "metadata": {
        "id": "WQlPfe_BGJ-k"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Load requirement data here"
      ],
      "metadata": {
        "id": "L5y7VP4PwChp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "dataset_version = '2.4.0'"
      ],
      "metadata": {
        "id": "W1EsEI-SF93O"
      },
      "execution_count": 62,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "### READ FILE HERE ###\n",
        "books_isbn = pd.read_csv(\n",
        "    '/content/drive/MyDrive/Chaloklum bookshop dataset/books' + dataset_version + '.txt',  \n",
        "    names=['Barcode'])"
      ],
      "metadata": {
        "id": "kX8m1PSVF4R_"
      },
      "execution_count": 64,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "### Open last version of books dataset ###\n",
        "dfBooks = pd.read_csv(\n",
        "    '/content/drive/MyDrive/Chaloklum bookshop dataset/v_2.3.5/books_dataset_w_review_2.3.5_final.csv')"
      ],
      "metadata": {
        "id": "x-WNI6kpI7Bp"
      },
      "execution_count": 66,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Call functions here"
      ],
      "metadata": {
        "id": "K8Axn8-svqaT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#books_isbn"
      ],
      "metadata": {
        "id": "lc812HwpGEgf"
      },
      "execution_count": 80,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "### CALL FUNCTIONS HERE ###"
      ],
      "metadata": {
        "id": "xrAXg1eRHACX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "newBooks(books_isbn)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        },
        "id": "zuboK-BRHCs7",
        "outputId": "fdde6946-3c92-412a-913e-57d560e7c4c5"
      },
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "18"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "' no book found: '"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "'8935036679840'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "retryAPIout(apiRestBooks)"
      ],
      "metadata": {
        "id": "inQ4hw7jHFbp"
      },
      "execution_count": 70,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "addGRbooks(noBooksList)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "id": "A-3IbuVQH1Zh",
        "outputId": "49af65bb-bb68-4bde-9881-136fd24fce97"
      },
      "execution_count": 71,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "['CHECKING >>> ', '8935036679840']"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "['NO INFO BOOKS: ', 11, ' LEFT']"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "['404 REQUEST: ', 0]"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "save_every_n = 100\n",
        "current_n = 0\n",
        "batch_title = 0\n",
        "file_type = '.csv'"
      ],
      "metadata": {
        "id": "nkzLge08Mvf8"
      },
      "execution_count": 72,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "getMoreData(dfBooks)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1Fe6VJnHIuXv",
        "outputId": "04375c86-1a94-4f75-cb15-63c8ef05d4ea"
      },
      "execution_count": 73,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "dfBooks at batch ...  0\n",
            "CURRENTLY  [ 15 / 100 ]\n",
            "NOW AT INDEX ...  3929 / 3937\n",
            "0  JSON ERROR\n",
            "GETTING CONTENT >>>  9781848875197\n",
            "GETTING TEXT REVIEWS >>>  30  REVIEWS\n",
            "GETTING DESCRIPTION ...\n",
            "GETTING ALSO ENJOYED BOOKS ...\n",
            "GETTING THUMBNAIL ...\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "path = '/content/drive/MyDrive/Chaloklum bookshop dataset/v_' + dataset_version + '/books_dataset_w_review_' + dataset_version + '_final' + file_type\n",
        "dfBooks.to_csv(path, index=False)"
      ],
      "metadata": {
        "id": "IlzcYiQCIvAW"
      },
      "execution_count": 75,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dfBooks"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "cli6c8vgNJRu",
        "outputId": "318ff4ed-a747-40de-d43a-72782478a30a"
      },
      "execution_count": 74,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "        id     isbn_10        isbn_13 isbn_other      isbn_book  \\\n",
              "0        1         NaN            NaN        NaN  9788020609564   \n",
              "1        2         NaN            NaN        NaN  9789637253089   \n",
              "2        3  068808835X  9780688088354        NaN     068808835X   \n",
              "3        4  068911611X  9780689116117        NaN     068911611X   \n",
              "4        5         NaN            NaN        NaN  9780316914454   \n",
              "...    ...         ...            ...        ...            ...   \n",
              "3932  3933        None  8710425001033        NaN  8710425001033   \n",
              "3933  3934  9604695789  9789604695782        NaN  9789604695782   \n",
              "3934  3935        None  9789604696079        NaN  9789604696079   \n",
              "3935  3936        None  9789604695799        NaN  9789604695799   \n",
              "3936  3937  9604102176  9789604102174       None  9789604102174   \n",
              "\n",
              "                                    authors  \\\n",
              "0                             ['Jiří Šolc']   \n",
              "1                                       NaN   \n",
              "2                          ['Anne Edwards']   \n",
              "3                            ['Bob Greene']   \n",
              "4                         ['Andrew Davies']   \n",
              "...                                     ...   \n",
              "3932                       [Barbara Pierce]   \n",
              "3933     [Agatha Christie, Λουκάς Λοράνδος]   \n",
              "3934  [Agatha Christie, Άννα Παπαδημητρίου]   \n",
              "3935   [Agatha Christie, Αχιλλέας Κυριαζής]   \n",
              "3936   [Paullina Simons, Γιάννης Σπανδωνής]   \n",
              "\n",
              "                                                  title  \\\n",
              "0                     Útěky a návraty Bohumila Laušmana   \n",
              "1                         A nők tartják az égbolt felét   \n",
              "2                                                Wallis   \n",
              "3                 Cheeseburgers, the Best of Bob Greene   \n",
              "4                                        We, the Nation   \n",
              "...                                                 ...   \n",
              "3932                 Troostende armen by Barbara Pierce   \n",
              "3933  Ποιος σκότωσε τον Ακρόυντ (Hercule Poirot, #4)...   \n",
              "3934  Ρετρό στην ομίχλη (Miss Marple, #13) by Agatha...   \n",
              "3935        Ο σατανάς με την πλερέζα by Agatha Christie   \n",
              "3936        O μπρούντζινος καβαλάρης by Paullina Simons   \n",
              "\n",
              "                                             subtitle  \\\n",
              "0                               osud českého politika   \n",
              "1     egy rendkívüli asszony rendkívüli élettörténete   \n",
              "2                                           The Novel   \n",
              "3                                                 NaN   \n",
              "4     The Conservative Party and the Pursuit of Power   \n",
              "...                                               ...   \n",
              "3932                                             None   \n",
              "3933                                             None   \n",
              "3934                                             None   \n",
              "3935                                             None   \n",
              "3936                                             None   \n",
              "\n",
              "                     publisher published_date  ... rating_avg #reviews  \\\n",
              "0                          NaN           2008  ...        NaN      NaN   \n",
              "1                          NaN           2005  ...        NaN      NaN   \n",
              "2     William Morrow & Company           1991  ...       3.74    312.0   \n",
              "3                     Scribner           1985  ...       4.11    165.0   \n",
              "4                          NaN           1995  ...        NaN      NaN   \n",
              "...                        ...            ...  ...        ...      ...   \n",
              "3932                      None           None  ...       3.52      187   \n",
              "3933                      None           None  ...       4.26   411472   \n",
              "3934                      None           None  ...       3.97    70692   \n",
              "3935                      None           None  ...       3.90    68117   \n",
              "3936                      None           None  ...       4.30   193065   \n",
              "\n",
              "     #ratings #text_reviews  \\\n",
              "0         NaN           NaN   \n",
              "1         NaN           NaN   \n",
              "2       151.0          22.0   \n",
              "3        93.0           9.0   \n",
              "4         NaN           NaN   \n",
              "...       ...           ...   \n",
              "3932       52             4   \n",
              "3933   222766         17339   \n",
              "3934    38677          2313   \n",
              "3935    35194          2066   \n",
              "3936    79196          8575   \n",
              "\n",
              "                                              thumbnail  \\\n",
              "0                                                   NaN   \n",
              "1                                                   NaN   \n",
              "2     http://books.google.com/books/content?id=wOyMH...   \n",
              "3     http://books.google.com/books/content?id=3ssiA...   \n",
              "4                                                   NaN   \n",
              "...                                                 ...   \n",
              "3932                                               None   \n",
              "3933                                               None   \n",
              "3934                                               None   \n",
              "3935                                               None   \n",
              "3936                                               None   \n",
              "\n",
              "                                                  genre  \\\n",
              "0                                                   NaN   \n",
              "1                                                   NaN   \n",
              "2     ['Historical', 'Historical Fiction', 'Fiction'...   \n",
              "3               ['Nonfiction', 'Writing', 'Journalism']   \n",
              "4                                                   NaN   \n",
              "...                                                 ...   \n",
              "3932                                                NaN   \n",
              "3933                                                NaN   \n",
              "3934                                                NaN   \n",
              "3935                                                NaN   \n",
              "3936                                                NaN   \n",
              "\n",
              "                                         goodreads_desc  \\\n",
              "0                                                   NaN   \n",
              "1                                                   NaN   \n",
              "2     \\nBorn as a poor relation to the wealthy Warfi...   \n",
              "3     \\nAmusing, poignant, and often surprising, Che...   \n",
              "4                                                   NaN   \n",
              "...                                                 ...   \n",
              "3932                                                NaN   \n",
              "3933                                                NaN   \n",
              "3934                                                NaN   \n",
              "3935                                                NaN   \n",
              "3936                                                NaN   \n",
              "\n",
              "                                           text_reviews  \\\n",
              "0                                                   NaN   \n",
              "1                                                   NaN   \n",
              "2     [[5.0, \"NetGalley provided me this book file i...   \n",
              "3     [[4.0, \"I don&apost remember all his essays, b...   \n",
              "4                                                   NaN   \n",
              "...                                                 ...   \n",
              "3932                                                NaN   \n",
              "3933                                                NaN   \n",
              "3934                                                NaN   \n",
              "3935                                                NaN   \n",
              "3936                                                NaN   \n",
              "\n",
              "                                             also_enjoy status  \n",
              "0                                                   NaN   -1.0  \n",
              "1                                                   NaN   -1.0  \n",
              "2     [['The Glorious Guinness Girls', 'https://i.gr...    1.0  \n",
              "3     [['Veeck As In Wreck: The Autobiography of Bil...    1.0  \n",
              "4                                                   NaN   -1.0  \n",
              "...                                                 ...    ...  \n",
              "3932                                                NaN    NaN  \n",
              "3933                                                NaN    NaN  \n",
              "3934                                                NaN    NaN  \n",
              "3935                                                NaN    NaN  \n",
              "3936                                                NaN    NaN  \n",
              "\n",
              "[3937 rows x 24 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-83fa7592-6569-42f0-b69a-6ad2e85da49c\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>isbn_10</th>\n",
              "      <th>isbn_13</th>\n",
              "      <th>isbn_other</th>\n",
              "      <th>isbn_book</th>\n",
              "      <th>authors</th>\n",
              "      <th>title</th>\n",
              "      <th>subtitle</th>\n",
              "      <th>publisher</th>\n",
              "      <th>published_date</th>\n",
              "      <th>...</th>\n",
              "      <th>rating_avg</th>\n",
              "      <th>#reviews</th>\n",
              "      <th>#ratings</th>\n",
              "      <th>#text_reviews</th>\n",
              "      <th>thumbnail</th>\n",
              "      <th>genre</th>\n",
              "      <th>goodreads_desc</th>\n",
              "      <th>text_reviews</th>\n",
              "      <th>also_enjoy</th>\n",
              "      <th>status</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>9788020609564</td>\n",
              "      <td>['Jiří Šolc']</td>\n",
              "      <td>Útěky a návraty Bohumila Laušmana</td>\n",
              "      <td>osud českého politika</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2008</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>-1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>9789637253089</td>\n",
              "      <td>NaN</td>\n",
              "      <td>A nők tartják az égbolt felét</td>\n",
              "      <td>egy rendkívüli asszony rendkívüli élettörténete</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2005</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>-1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>068808835X</td>\n",
              "      <td>9780688088354</td>\n",
              "      <td>NaN</td>\n",
              "      <td>068808835X</td>\n",
              "      <td>['Anne Edwards']</td>\n",
              "      <td>Wallis</td>\n",
              "      <td>The Novel</td>\n",
              "      <td>William Morrow &amp; Company</td>\n",
              "      <td>1991</td>\n",
              "      <td>...</td>\n",
              "      <td>3.74</td>\n",
              "      <td>312.0</td>\n",
              "      <td>151.0</td>\n",
              "      <td>22.0</td>\n",
              "      <td>http://books.google.com/books/content?id=wOyMH...</td>\n",
              "      <td>['Historical', 'Historical Fiction', 'Fiction'...</td>\n",
              "      <td>\\nBorn as a poor relation to the wealthy Warfi...</td>\n",
              "      <td>[[5.0, \"NetGalley provided me this book file i...</td>\n",
              "      <td>[['The Glorious Guinness Girls', 'https://i.gr...</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>068911611X</td>\n",
              "      <td>9780689116117</td>\n",
              "      <td>NaN</td>\n",
              "      <td>068911611X</td>\n",
              "      <td>['Bob Greene']</td>\n",
              "      <td>Cheeseburgers, the Best of Bob Greene</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Scribner</td>\n",
              "      <td>1985</td>\n",
              "      <td>...</td>\n",
              "      <td>4.11</td>\n",
              "      <td>165.0</td>\n",
              "      <td>93.0</td>\n",
              "      <td>9.0</td>\n",
              "      <td>http://books.google.com/books/content?id=3ssiA...</td>\n",
              "      <td>['Nonfiction', 'Writing', 'Journalism']</td>\n",
              "      <td>\\nAmusing, poignant, and often surprising, Che...</td>\n",
              "      <td>[[4.0, \"I don&amp;apost remember all his essays, b...</td>\n",
              "      <td>[['Veeck As In Wreck: The Autobiography of Bil...</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>9780316914454</td>\n",
              "      <td>['Andrew Davies']</td>\n",
              "      <td>We, the Nation</td>\n",
              "      <td>The Conservative Party and the Pursuit of Power</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1995</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>-1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3932</th>\n",
              "      <td>3933</td>\n",
              "      <td>None</td>\n",
              "      <td>8710425001033</td>\n",
              "      <td>NaN</td>\n",
              "      <td>8710425001033</td>\n",
              "      <td>[Barbara Pierce]</td>\n",
              "      <td>Troostende armen by Barbara Pierce</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>...</td>\n",
              "      <td>3.52</td>\n",
              "      <td>187</td>\n",
              "      <td>52</td>\n",
              "      <td>4</td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3933</th>\n",
              "      <td>3934</td>\n",
              "      <td>9604695789</td>\n",
              "      <td>9789604695782</td>\n",
              "      <td>NaN</td>\n",
              "      <td>9789604695782</td>\n",
              "      <td>[Agatha Christie, Λουκάς Λοράνδος]</td>\n",
              "      <td>Ποιος σκότωσε τον Ακρόυντ (Hercule Poirot, #4)...</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>...</td>\n",
              "      <td>4.26</td>\n",
              "      <td>411472</td>\n",
              "      <td>222766</td>\n",
              "      <td>17339</td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3934</th>\n",
              "      <td>3935</td>\n",
              "      <td>None</td>\n",
              "      <td>9789604696079</td>\n",
              "      <td>NaN</td>\n",
              "      <td>9789604696079</td>\n",
              "      <td>[Agatha Christie, Άννα Παπαδημητρίου]</td>\n",
              "      <td>Ρετρό στην ομίχλη (Miss Marple, #13) by Agatha...</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>...</td>\n",
              "      <td>3.97</td>\n",
              "      <td>70692</td>\n",
              "      <td>38677</td>\n",
              "      <td>2313</td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3935</th>\n",
              "      <td>3936</td>\n",
              "      <td>None</td>\n",
              "      <td>9789604695799</td>\n",
              "      <td>NaN</td>\n",
              "      <td>9789604695799</td>\n",
              "      <td>[Agatha Christie, Αχιλλέας Κυριαζής]</td>\n",
              "      <td>Ο σατανάς με την πλερέζα by Agatha Christie</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>...</td>\n",
              "      <td>3.90</td>\n",
              "      <td>68117</td>\n",
              "      <td>35194</td>\n",
              "      <td>2066</td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3936</th>\n",
              "      <td>3937</td>\n",
              "      <td>9604102176</td>\n",
              "      <td>9789604102174</td>\n",
              "      <td>None</td>\n",
              "      <td>9789604102174</td>\n",
              "      <td>[Paullina Simons, Γιάννης Σπανδωνής]</td>\n",
              "      <td>O μπρούντζινος καβαλάρης by Paullina Simons</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>...</td>\n",
              "      <td>4.30</td>\n",
              "      <td>193065</td>\n",
              "      <td>79196</td>\n",
              "      <td>8575</td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>3937 rows × 24 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-83fa7592-6569-42f0-b69a-6ad2e85da49c')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-83fa7592-6569-42f0-b69a-6ad2e85da49c button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-83fa7592-6569-42f0-b69a-6ad2e85da49c');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 74
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dfBooks_reorder = dfBooks[['id', 'title', 'subtitle', 'authors', 'isbn_10', 'isbn_13', 'isbn_other', 'isbn_book', 'language', 'google_desc', 'categories', 'goodreads_desc', 'publisher', 'published_date', 'page_count', 'rating_avg', '#reviews', '#ratings', '#text_reviews', 'thumbnail', 'genre', 'text_reviews', 'also_enjoy', 'status']]"
      ],
      "metadata": {
        "id": "Qfw8m00KQYpy"
      },
      "execution_count": 76,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "path_reorder = '/content/drive/MyDrive/Chaloklum bookshop dataset/v_' + dataset_version + '/books_dataset_w_review_' + dataset_version + '_reordered' + file_type\n",
        "dfBooks_reorder.to_csv(path_reorder, index=False)"
      ],
      "metadata": {
        "id": "397azduYSaEV"
      },
      "execution_count": 77,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dfBooks_for_finder = dfBooks_reorder.drop(columns=['text_reviews', 'thumbnail', 'also_enjoy', 'status', '#reviews',\t'#ratings',\t'#text_reviews', 'isbn_10', 'isbn_13', 'isbn_other'])"
      ],
      "metadata": {
        "id": "dw_zNW2qS4PG"
      },
      "execution_count": 78,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "path_for_finder = '/content/drive/MyDrive/Chaloklum bookshop dataset/v_' + dataset_version + '/books_dataset_' + dataset_version + '_for_finder' + file_type\n",
        "dfBooks_for_finder.to_csv(path_for_finder, index=False)"
      ],
      "metadata": {
        "id": "gLfF6RF4TyOQ"
      },
      "execution_count": 79,
      "outputs": []
    }
  ]
}