mining_ood_twitter.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package vader_lexicon to\n",
      "[nltk_data]     /home/ishalyminov/nltk_data...\n",
      "[nltk_data]   Package vader_lexicon is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "import os\n",
    "from operator import itemgetter\n",
    "\n",
    "from utils.twitter import download_data\n",
    "from utils.nlp_utils import is_positive, is_negative, profanity_check, contains_nes, contains_blacklisted_pos_tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_FOLDER = 'data'\n",
    "TWITTER_DATASET_FILENAME = 'twitter_en_big.txt'\n",
    "if not os.path.exists(DATA_FOLDER):\n",
    "    os.makedirs(DATA_FOLDER)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "FULL_FILENAME = os.path.join(DATA_FOLDER, TWITTER_DATASET_FILENAME)\n",
    "\n",
    "if not os.path.exists(FULL_FILENAME):\n",
    "    download_data(FULL_FILENAME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(DATA_FOLDER, TWITTER_DATASET_FILENAME), 'r', encoding='utf-8') as twitter_in:\n",
    "    lines = list(set(map(lambda x: x.lower().strip(), twitter_in.readlines())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# unique utterances: 4691739\n"
     ]
    }
   ],
   "source": [
    "print('# unique utterances: {}'.format(len(lines)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def utterance_is_ok(in_utterance):\n",
    "    return (not contains_blacklisted_pos_tags(in_utterance)) \\\n",
    "           and profanity_check(in_utterance) \\\n",
    "           and (not contains_nes(in_utterance))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "positive_utterances = [utterance for utterance in lines\n",
    "                       if is_positive(utterance, positive_threshold=0.9, negative_threshold=0.1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "positive_utterances = list(filter(lambda x: utterance_is_ok(x), positive_utterances))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# positive utterances after filtering: 2468\n",
      "encouraging\n",
      "winning. :)\n",
      "lol alright alright\n",
      "congrats! welcome\n",
      "amazing textures! good luck!\n",
      "interesting, thanks.\n",
      "honestly truth\n",
      "i support u\n",
      "super helpful! thanks!\n",
      "- congrats!\n",
      "yes! yes! wow\n",
      "yay my favorite holiday\n",
      "wow!!! absolutely amazing!\n",
      "thanks charity\n",
      "yes!!! well deserved!!! congrats!\n",
      "ha hah ha ha ha\n",
      "yes!! please share!\n",
      "yes, thanks\n",
      "thanks, friend.\n",
      "thanks friends\n",
      "awesome :-d\n",
      "definitely cute!\n",
      "thanks joy!\n",
      "cheers! love\n",
      "wow good deal! ok\n",
      "love, love!!\n",
      "awesome!! thanks :)\n",
      "honest, raw, truthful, proud, survivor,\n",
      "awesome kindness!\n",
      "- congrats. great school great love\n",
      "haha yes ! 😋\n",
      "amazing. congrats.\n",
      "thanks rich!\n",
      "yeah, she's special alright. thanks!\n",
      "positivity 💜\n",
      "beautiful!! i love architecture!!\n",
      "wow! wonderful!\n",
      "yes. good luck!\n",
      "cheers! support . . . .\n",
      "wow she's talented. thanks .\n",
      "you're welcome lovely! good luck! :*\n",
      "yay! vindicated!\n",
      "ha!! is pretty good.\n",
      "excellent - thanks!\n",
      "also love love love\n",
      "thanks gorgeous\n",
      "congratulations, !\n",
      "yes, thankfully. :-)\n",
      "wow gorgeous 💛\n",
      "yes good charity yes\n",
      "congratulations!!! :-)\n",
      "yes please lol\n",
      "love that win. nice\n",
      "congratulations! wonderful!\n",
      "haha yes dear :p\n",
      "thanks!! they're pretty nice.\n",
      "pretty good thanks\n",
      "wow - thank you :)\n",
      "thanks, . :)\n",
      "nice, good luck!\n",
      "congratulations winner …\n",
      "yeah, that's pretty awesome!\n",
      "pls pray :)\n",
      "astoundingly great!\n",
      "wow!! love it!!\n",
      "wonderful!!\n",
      "pretty well\n",
      "i appreciate. 😘\n",
      "haha yeah everyone!!! lol!!\n",
      "amaze!!! congrats\n",
      "i agree. love, love, love birchbox\n",
      "okay thanks!\n",
      "👀 playing nice?\n",
      "wonderful !\n",
      "hope this helps lmao\n",
      "thanks lovely!!!\n",
      "yes!!! congrats! well deserved!!! 😀\n",
      "definitely 🚀\n",
      "love ! awesome\n",
      "yes yes yes.\n",
      "yep! thanks!\n",
      "mercy progress :d\n",
      "beautiful friend\n",
      "wow okay love doctor\n",
      "well played.\n",
      "awesome thanks.\n",
      "yeah yeah 😴\n",
      "alright lol\n",
      "yes. better.\n",
      "please play please\n",
      "pretty funny\n",
      "haha you won!!! congrats\n",
      "thanks dear 😊\n",
      "cool - thanks.\n",
      "alright fair.\n",
      "amazing!! congratulations!\n",
      "wow beautiful\n",
      "yes, agreed.\n",
      "best of luck! \\o/\n",
      "best friend easily\n"
     ]
    }
   ],
   "source": [
    "print('# positive utterances after filtering: {}'.format(len(positive_utterances)))\n",
    "print('\\n'.join(positive_utterances[:100]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(DATA_FOLDER, 'twitter_ood_positive.txt'), 'w') as positive_out:\n",
    "    for utterance in positive_utterances:\n",
    "        print(utterance, file=positive_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "negative_utterances = [utterance for utterance in lines\n",
    "                       if is_negative(utterance, positive_threshold=0.3, negative_threshold=0.8)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "negative_utterances = list(filter(lambda x: utterance_is_ok(x), negative_utterances))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# negative utterances after filtering: 1449\n",
      "disgusting and murderous!\n",
      "stop embarrassing yourself\n",
      "what's worst?\n",
      "ridiculous! a non-issue, a distraction\n",
      "what's wrong? :(\n",
      "blocking this fool.\n",
      "stop i might cry\n",
      "obsessed!!!\n",
      "fail. pathetic.\n",
      "ugh what a nerd\n",
      "my fault :/\n",
      "no scandal here\n",
      "that's horrendous!! those poor terrified horses!!\n",
      "ugh that's absolutely awful :/\n",
      "stop i hate it\n",
      "nothing like optimism\n",
      "low key sad\n",
      "strangely, no.\n",
      "obsessed with shameless\n",
      "tired and irritated\n",
      ":( eeeh sorry!!\n",
      "no tears. none.\n",
      "pay up fool\n",
      "sorry, i misunderstood.\n",
      "anxiety is exhausting.\n",
      "a sad, sad, situation! 😢\n",
      "hopelessness.\n",
      "seriously ?\n",
      "ugh! what a witch\n",
      "sarcastic or no\n",
      "wrong !!!!!!!!!!!!!!!!!!!!!!!!\n",
      "foolishness\n",
      "outrageous!!!\n",
      "so terribly sad :(\n",
      "so sad. terrorists reign!\n",
      "i predict fatality.\n",
      "unfortunately probably puking 😞\n",
      "no pictures? :(\n",
      "... hate? seriously, hate? what?\n",
      "doubtful, very doubtful.\n",
      "disgusting puppets!!!!\n",
      "haters are looming\n",
      "r u complaining\n",
      "wicked wicked wicked wicked\n",
      "horrible and horrifying\n",
      "serious crush!! 😘 serious!!\n",
      "cries in freak\n",
      "he's a freaking dork\n",
      "liars liars liars liars liars\n",
      "more dirt exposed!\n",
      "phony horrible sickening\n",
      "wait wth seriously ?\n",
      "awful awful call\n",
      "sad, scared, worried, terrified, lost, and the lost goes on.\n",
      ". i hate haters. hate 'em, hate 'em, hate 'em.\n",
      "no problem, rockstar\n",
      "ugh this sucks.\n",
      "they disgust me, boring losers\n",
      "ineffective?\n",
      "a troubled loner, eh?\n",
      "i hate you sorry\n",
      "disgusting, and appalling 💔\n",
      "heartbreaking :(\n",
      "bad meaning terrible\n",
      "stop whining 🙄\n",
      "exhausting.\n",
      "no argument there\n",
      "ugh i miss those!\n",
      "thats evil smh\n",
      "u so petty wth\n",
      "i seriously doubt it\n",
      "no, that's crazy!!!\n",
      "sad??? it's pathetic!\n",
      "total disgust!\n",
      "miss y'all :(\n",
      "no, you're crying\n",
      "chokes worse than\n",
      "disgusting shameful and life threatening\n",
      "stop bullying me\n",
      "evil. never. wins.\n",
      "that sucks :(\n",
      "that's vile\n",
      "sorry sorry sorry sorry\n",
      "ugh!!! i never win\n",
      "stressed. stressed. stressed. stressed. stressed. stressed. stressed. stressed. stressed. stressed. stressed. stressed.\n",
      "i miss her :(\n",
      "no complaints\n",
      "no youre wrong\n",
      "stop. no. what?\n",
      "ugh that's annoying :( :(\n",
      "ugh. so awful.\n",
      "terrible. just terrible.\n",
      "- intimidation!\n",
      "i pity the fool\n",
      "no, it’s racism …\n",
      "scandalous.\n",
      "dangerously miserable.\n",
      "stop the negativity :/\n",
      "that's a serious shame.\n",
      "ugh! disgust!!!\n"
     ]
    }
   ],
   "source": [
    "print('# negative utterances after filtering: {}'.format(len(negative_utterances)))\n",
    "print('\\n'.join(negative_utterances[:100]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(DATA_FOLDER, 'twitter_ood_negative.txt'), 'w') as negative_out:\n",
    "    for utterance in negative_utterances:\n",
    "        print(utterance, file=negative_out)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}