-
Notifications
You must be signed in to change notification settings - Fork 0
/
process.py
29 lines (22 loc) · 956 Bytes
/
process.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
from nltk.tokenize import word_tokenize, sent_tokenize
import re
import string
import nltk
from nltk.corpus import stopwords
import pymorphy2
#nltk.download('stopwords')
stopWords = set(stopwords.words("russian"))
m = pymorphy2.MorphAnalyzer(lang='ru')
def remove_numbers(text):
return ''.join([i if not i.isdigit() else ' ' for i in text])
def remove_punctuation(text):
punc = "!\"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~»«—–-"
return "".join([ch if ch not in punc else ' ' for ch in text])
def remove_multiple_spaces(text):
return re.sub(r'\s+', ' ', text, flags=re.I)
def get_normal_form(text):
return ' '.join([m.parse(w)[0].normal_form for w in word_tokenize(text)])
def delete_stopwords(text):
return ' '.join([(word) for word in word_tokenize(text) if word not in stopWords])
def preproccessing(text):
return remove_multiple_spaces(delete_stopwords(get_normal_form(remove_numbers(remove_punctuation(text.strip())))))