-
Notifications
You must be signed in to change notification settings - Fork 0
/
hashmap_file_singlethread.py
51 lines (38 loc) · 1.45 KB
/
hashmap_file_singlethread.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
import time
import re
import logging as log
import os
def read_stop_words(file_path):
with open(file_path, "r", encoding="utf-8-sig") as sw:
stop_words = set()
for line in sw:
stop_words.update(line.strip().split(","))
return stop_words
def read_datafile(file_path, stop_words):
word_counts = dict()
with open(file_path, "r", encoding="utf-8-sig") as file:
for line in file:
for word in re.findall(r"\w+", line.lower()):
if word and word not in stop_words:
word_counts[word] = word_counts.get(word,0) + 1
return word_counts
def main():
# set the number of top words to find
k = int(input("Enter the number of top words to find: "))
filename = log.FILENAME_16GB
# get the file size
file_name = "/Users/rushshah/SCU/BigData/" + filename
file_size = os.path.getsize(file_name)
file_size_GB = file_size / (1024 * 1024 * 1024)
# read stop words from file
stop_words = read_stop_words(log.FILE_STOP_WORDS)
start_time = time.time()
# count words from data file
word_counts = read_datafile(file_name, stop_words)
current_file = os.path.basename(__file__).split(".")[0]
# print the top k frequent words
log.get_top_words_hashmap(word_counts, k)
# print the performance statistics
log.print_statistics(current_file, filename, start_time, file_size_GB, None, k)
if __name__ == '__main__':
main()