n-grams-and-collacations-on-tbmm-corpus.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Initializing\n\n!pip install nltk\n!pip install -U textblob \n!python -m textblob.download_corpora\n!pip install nltk\n!pip install openpyxl\n\nimport sys\nimport pandas as pd\nimport xlwt\nimport csv\nimport numpy as np\nfrom numpy import loadtxt\nfrom textblob import TextBlob\nfrom collections import Counter\n\nimport nltk, re, string, collections\nfrom nltk import word_tokenize \nfrom nltk.corpus import stopwords\nnltk.download('stopwords')\nfrom nltk.util import ngrams","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:09:40.965242Z","iopub.execute_input":"2021-05-22T23:09:40.965868Z","iopub.status.idle":"2021-05-22T23:10:14.403567Z","shell.execute_reply.started":"2021-05-22T23:09:40.965728Z","shell.execute_reply":"2021-05-22T23:10:14.401274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def readFile(fileName):\n  file = open(fileName, \"r\")\n  words = file.read()\n  file.close\n  return words  ","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:11:06.354886Z","iopub.execute_input":"2021-05-22T23:11:06.355406Z","iopub.status.idle":"2021-05-22T23:11:06.361918Z","shell.execute_reply.started":"2021-05-22T23:11:06.355362Z","shell.execute_reply":"2021-05-22T23:11:06.360545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Filtering\n\nfor x in [\"20\",\"21\",\"22\",\"23\",\"24\",\"25\",\"26\",\"27\"]:\n\n  # appendFile = open('filtered.txt','a+')\n\n  path = \"filtered-corpus/filtered-donem\" + x + \".txt\"\n  all_files = glob.glob(path)\n  words = \"\"\n  \n  # appendFile = open('filtered-corpus/filtered-donem' + x + '.txt','a+')\n\n  for file in all_files:\n    words = words + readFile(file)\n  \n  tokenized = [idx.lower() for idx in words.split() if re.findall(\"^[a-zA-ZğüşöçİĞÜŞÖÇ]+$\", idx) and len(idx) > 1]\n\n  # for idx in tokenized:\n  #   appendFile.write(\" \"+idx.lower())\n  \n  # appendFile.close()","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:11:06.364568Z","iopub.execute_input":"2021-05-22T23:11:06.365033Z","iopub.status.idle":"2021-05-22T23:11:06.388471Z","shell.execute_reply.started":"2021-05-22T23:11:06.364983Z","shell.execute_reply":"2021-05-22T23:11:06.386938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# n-grams\n\nfor x in [\"20\",\"21\",\"22\",\"23\",\"24\",\"25\",\"26\",\"27\"]:\n\n  path = \"corpus/donem\" + x + \"/yıl*/*.txt\"\n  all_files = glob.glob(path)\n  words = \"\"\n  \n  for file in all_files:\n    words = words + readFile(file)\n  \n  tokenized = [idx.lower() for idx in words.split() if re.findall(\"^[a-zA-Z0-9ğüşöçİĞÜŞÖÇ]+$\", idx) and len(idx) > 1]\n\n  unigrams = ngrams(tokenized, 1)\n  unigramsFreq = collections.Counter(unigrams)\n  df = pd.DataFrame(unigramsFreq.most_common(1000), columns = ['Words', 'Score'])\n  df = df.sort_values(by='Score', ascending=False)\n  p = '.results-corpus/without-stopwords/unigrams/unigram'+ x +'.csv'\n  df.to_csv(p, index = True, header=True)\n\n  bigrams = ngrams(tokenized, 2)\n  bigramsFreq = collections.Counter(bigrams)\n  df = pd.DataFrame(bigramsFreq.most_common(1000), columns = ['Words', 'Score'])\n  df = df.sort_values(by='Score', ascending=False)\n  p = '.results-corpus/without-stopwords/bigrams/bigram'+ x +'.csv'\n  df.to_csv(p, index = True, header=True)\n\n  trigrams = ngrams(tokenized, 3)\n  trigramsFreq = collections.Counter(trigrams)\n  df = pd.DataFrame(trigramsFreq.most_common(1000), columns = ['Words', 'Score'])\n  df = df.sort_values(by='Score', ascending=False)\n  p = '.results-corpus/without-stopwords/trigrams/trigram'+ x +'.csv'\n  df.to_csv(p, index = True, header=True)","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:11:06.389447Z","iopub.status.idle":"2021-05-22T23:11:06.389865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Collocations\n\ncorpus = readFile('filtered-corpus/all-corpus-for-collacations.txt')\n\ntokenized = [idx.lower() for idx in corpus.split()]\n# tokenized = [item for items in corpus for item in items]\n\nbigrams = nltk.collocations.BigramAssocMeasures()\ntrigrams = nltk.collocations.TrigramAssocMeasures()\n\nbigramFinder = nltk.collocations.BigramCollocationFinder.from_words(tokenized)\ntrigramFinder = nltk.collocations.TrigramCollocationFinder.from_words(tokenized)\n\nbigramFinder.apply_freq_filter(30)\ntrigramFinder.apply_freq_filter(30)\n\nbigram_freq = bigramFinder.ngram_fd.items()\nbigramFreqTable = pd.DataFrame(list(bigram_freq), columns=['bigram','freq']).sort_values(by='freq', ascending=False)\n# bigramFreqTable[:10]\n\ntrigram_freq = trigramFinder.ngram_fd.items()\ntrigramFreqTable = pd.DataFrame(list(trigram_freq), columns=['trigram','freq']).sort_values(by='freq', ascending=False)\n# trigramFreqTable[:10] ","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:11:06.39093Z","iopub.status.idle":"2021-05-22T23:11:06.391562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# PMI\n\nbigramPMITable = pd.DataFrame(list(bigramFinder.score_ngrams(bigrams.pmi)), columns=['bigram','PMI']).sort_values(by='PMI', ascending=False)\n# bigramPMITable[:10]\np = 'collacations-results/all-corpus-bigram-pmi.csv'\nbigramPMITable[:100].to_csv(p, index = True, header=True)","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:11:06.393158Z","iopub.status.idle":"2021-05-22T23:11:06.393836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trigramPMITable = pd.DataFrame(list(trigramFinder.score_ngrams(trigrams.pmi)), columns=['trigram','PMI']).sort_values(by='PMI', ascending=False)\n# trigramPMITable[:10]\np = 'collacations-results/all-corpus-trigram-pmi.csv'\ntrigramPMITable[:100].to_csv(p, index = True, header=True)","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:11:06.394948Z","iopub.status.idle":"2021-05-22T23:11:06.395459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# t-test\n\nbigramTtable = pd.DataFrame(list(bigramFinder.score_ngrams(bigrams.student_t)), columns=['bigram','t']).sort_values(by='t', ascending=False)\n# bigramTtable[:10]\np = 'collacations-results/all-corpus-bigram-t-test.csv'\nbigramTtable[:100].to_csv(p, index = True, header=True)","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:11:06.396452Z","iopub.status.idle":"2021-05-22T23:11:06.396886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trigramTtable = pd.DataFrame(list(trigramFinder.score_ngrams(trigrams.student_t)), columns=['trigram','t']).sort_values(by='t', ascending=False)\n# trigramTtable[:10]\np = 'collacations-results/all-corpus-trigram-t-test.csv'\ntrigramTtable[:100].to_csv(p, index = True, header=True)","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:11:06.397819Z","iopub.status.idle":"2021-05-22T23:11:06.398266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Chi-Square\n\nbigramChiTable = pd.DataFrame(list(bigramFinder.score_ngrams(bigrams.chi_sq)), columns=['bigram','chi-sq']).sort_values(by='chi-sq', ascending=False)\n# bigramChiTable[:10]\np = 'collacations-results/all-corpus-bigram-chi.csv'\nbigramChiTable[:100].to_csv(p, index = True, header=True)","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:11:06.500504Z","iopub.execute_input":"2021-05-22T23:11:06.500998Z","iopub.status.idle":"2021-05-22T23:11:06.519108Z","shell.execute_reply.started":"2021-05-22T23:11:06.500952Z","shell.execute_reply":"2021-05-22T23:11:06.517339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trigramChiTable = pd.DataFrame(list(trigramFinder.score_ngrams(trigrams.chi_sq)), columns=['trigram','chi-sq']).sort_values(by='chi-sq', ascending=False)\n# trigramChiTable[:10]\np = 'collacations-results/all-corpus-trigram-chi.csv'\ntrigramChiTable[:100].to_csv(p, index = True, header=True)","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:11:06.520166Z","iopub.status.idle":"2021-05-22T23:11:06.520622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#comparison\n\nfreq_bi = bigramFreqTable[:15].bigram.values\nfreq_tri = trigramFreqTable[:15].trigram.values\n\npmi_bi = bigramPMITable[:15].bigram.values\npmi_tri = trigramPMITable[:15].trigram.values\n\nt_bi = bigramTtable[:15].bigram.values\nt_tri = trigramTtable[:15].trigram.values\n\nchi_bi = bigramChiTable[:15].bigram.values\nchi_tri = trigramChiTable[:15].trigram.values","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:11:06.522114Z","iopub.status.idle":"2021-05-22T23:11:06.522564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bigramsCompare = pd.DataFrame([freq_bi, pmi_bi, t_bi, chi_bi]).T\nbigramsCompare.columns = ['Frequency', 'PMI', 't-test', 'Chi-Sq Test']\n# bigramsCompare\np = 'collacations-results/all-corpus-bigram-compare.csv'\nbigramsCompare[:100].to_csv(p, index = True, header=True)","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:11:06.52357Z","iopub.status.idle":"2021-05-22T23:11:06.523997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trigramsCompare = pd.DataFrame([freq_tri, pmi_tri, t_tri, chi_tri]).T\ntrigramsCompare.columns = ['Frequency', 'PMI', 't-test', 'Chi-Sq Test']\n# trigramsCompare\np = 'collacations-results/all-corpus-trigram-compare.csv'\ntrigramsCompare[:100].to_csv(p, index = True, header=True)","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:11:06.525071Z","iopub.status.idle":"2021-05-22T23:11:06.525549Z"},"trusted":true},"execution_count":null,"outputs":[]}]}