Scrapping-Hindi-Sarcastic-Tweets.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Scrapping Hindi Sarcastic Tweets </h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tweepy import OAuthHandler\n",
    "from tweepy.streaming import StreamListener\n",
    "import tweepy\n",
    "import json\n",
    "import pandas as pd\n",
    "import csv\n",
    "import re\n",
    "from textblob import TextBlob\n",
    "import string\n",
    "import preprocessor as p\n",
    "import os\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Twitter credentials\n",
    "# Obtain them from your twitter developer account\n",
    "consumer_key = '---'\n",
    "consumer_secret = '---'\n",
    "access_key = '---'\n",
    "access_secret = '---'\n",
    "\n",
    "auth = OAuthHandler(consumer_key, consumer_secret)\n",
    "auth.set_access_token(access_key, access_secret)\n",
    "api = tweepy.API(auth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scraptweets(search_words, date_since, numTweets, numRuns):\n",
    "\n",
    "    db_tweets = pd.DataFrame(columns = ['username', 'acctdesc', 'location', 'following',\n",
    "                                        'followers', 'totaltweets', 'usercreatedts', 'tweetcreatedts',\n",
    "                                        'retweetcount', 'text', 'hashtags'])\n",
    "    \n",
    "    for i in range(0, numRuns):\n",
    "        start_run = time.time()\n",
    "        tweets = tweepy.Cursor(api.search, q=search_words, lang=\"hi\", since=date_since, tweet_mode='extended').items(numTweets)\n",
    "        tweet_list = [tweet for tweet in tweets]\n",
    "        noTweets = 0\n",
    "\n",
    "        for tweet in tweet_list:\n",
    "            username = tweet.user.screen_name\n",
    "            acctdesc = tweet.user.description\n",
    "            location = tweet.user.location\n",
    "            following = tweet.user.friends_count\n",
    "            followers = tweet.user.followers_count\n",
    "            totaltweets = tweet.user.statuses_count\n",
    "            usercreatedts = tweet.user.created_at\n",
    "            tweetcreatedts = tweet.created_at\n",
    "            retweetcount = tweet.retweet_count\n",
    "            hashtags = tweet.entities['hashtags']\n",
    "\n",
    "            try:\n",
    "                text = tweet.retweeted_status.full_text\n",
    "            except AttributeError:  # Not a Retweet\n",
    "                text = tweet.full_text\n",
    "\n",
    "            ith_tweet = [username, acctdesc, location, following, followers, totaltweets,\n",
    "                         usercreatedts, tweetcreatedts, retweetcount, text, hashtags]\n",
    "\n",
    "            db_tweets.loc[len(db_tweets)] = ith_tweet\n",
    "            noTweets += 1\n",
    "        \n",
    "        end_run = time.time()\n",
    "        duration_run = round(end_run-start_run, 2)\n",
    "        \n",
    "        print('no. of tweets scraped for run {} is {}'.format(i, noTweets))\n",
    "        print('time take for {} run to complete is {}'.format(i, duration_run))\n",
    "        \n",
    "        time.sleep(900) #15 minute sleep time\n",
    "\n",
    "        \n",
    "    from datetime import datetime\n",
    "    to_csv_timestamp = datetime.today().strftime('%Y%m%d_%H%M%S')\n",
    "\n",
    "    path = os.getcwd()\n",
    "    filename = 'Sarcasm_Hindi_Tweets.csv'\n",
    "\n",
    "    db_tweets.to_csv(filename, index = False)\n",
    "    \n",
    "    print('Scraping has completed!')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extracting Sarcastic Hindi Tweets:\n",
    "\n",
    "Extracted for the duration: 01-01-2012 to 23-06-2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no. of tweets scraped for run 0 is 604\n",
      "time take for 0 run to complete is 66.84\n",
      "no. of tweets scraped for run 1 is 605\n",
      "time take for 1 run to complete is 76.99\n",
      "no. of tweets scraped for run 2 is 605\n",
      "time take for 2 run to complete is 71.17\n",
      "no. of tweets scraped for run 3 is 605\n",
      "time take for 3 run to complete is 70.58\n",
      "no. of tweets scraped for run 4 is 605\n",
      "time take for 4 run to complete is 66.47\n",
      "no. of tweets scraped for run 5 is 605\n",
      "time take for 5 run to complete is 73.19\n",
      "no. of tweets scraped for run 6 is 605\n",
      "time take for 6 run to complete is 71.42\n",
      "no. of tweets scraped for run 7 is 605\n",
      "time take for 7 run to complete is 70.61\n",
      "no. of tweets scraped for run 8 is 606\n",
      "time take for 8 run to complete is 76.58\n",
      "no. of tweets scraped for run 9 is 606\n",
      "time take for 9 run to complete is 69.17\n",
      "Scraping has completed!\n"
     ]
    }
   ],
   "source": [
    "search_words = \"#कटाक्ष OR #kataksh OR #sarcasm OR #sarcastic OR #irony\"\n",
    "date_since = \"2012-01-01\"\n",
    "numTweets = 2500\n",
    "numRuns = 10\n",
    "\n",
    "scraptweets(search_words, date_since, numTweets, numRuns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extracting Non-Sarcastic Hindi Tweets:\n",
    "\n",
    "Extracted for the duration: 01-01-2012 to 22-06-2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no. of tweets scraped for run 0 is 2025\n",
      "time take for 0 run to complete is 296.76\n",
      "no. of tweets scraped for run 1 is 2016\n",
      "time take for 1 run to complete is 293.76\n",
      "no. of tweets scraped for run 2 is 2024\n",
      "time take for 2 run to complete is 251.52\n",
      "no. of tweets scraped for run 3 is 2028\n",
      "time take for 3 run to complete is 309.01\n",
      "no. of tweets scraped for run 4 is 2035\n",
      "time take for 4 run to complete is 262.67\n",
      "Scraping has completed!\n"
     ]
    }
   ],
   "source": [
    "search_words = \"#खुशी OR #खुशियाँ OR #happy OR #smile OR #sad OR #sadness OR #दुख OR #दुखी OR #उदास\"\n",
    "date_since = \"2012-01-01\"\n",
    "numTweets = 2500\n",
    "numRuns = 5\n",
    "\n",
    "scraptweets(search_words, date_since, numTweets, numRuns)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}