-
Notifications
You must be signed in to change notification settings - Fork 0
/
parsing_sentences.py
66 lines (51 loc) · 2.12 KB
/
parsing_sentences.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
# -*- coding: utf-8 -*-
import codecs
from lxml import etree
import json
import sys
names = {
'types': "http:///gr/ilsp/types.ecore",
'xmi': "http://www.omg.org/XMI"
}
n = {
'cas': "http:///uima/cas.ecore",
'xmi': "http://www.omg.org/XMI"
}
with codecs.open('sentences.json', 'w', encoding='utf-8', errors='ignore') as sentences:
for i in range(1696):
file = sys.argv[1]+'/archive/split' + str(i)+'_makedonia.txt'
print(file)
with codecs.open(file, 'r', encoding='utf-8', errors='ignore') as file:
article = etree.parse(file)
sofas = article.xpath(
"cas:Sofa/@sofaString",
namespaces=n
)[0] # news text
begin_tokens = article.xpath("//types:Token/@begin", namespaces=names)
end_tokens = article.xpath("//types:Token/@end", namespaces=names)
# index of token in sentence
sent_ord = article.xpath("//types:Token/@sentOrd", namespaces=names)
num_sentences = len(article.xpath(
"//types:Sentence/@end",
namespaces=names
))
# pos id to locate the class
pos_tokens_id = article.xpath(
"//types:Token/@posTag",
namespaces=names
)
index = 0
for num in range(num_sentences):
sentence = ''
while 1:
begin_token_index = int(begin_tokens[index])
end_token_index = int(end_tokens[index])
token_value = sofas[begin_token_index:end_token_index]
sentence = sentence + token_value + ' '
index = index + 1
if len(sent_ord) == index or sent_ord[index] == '1':
if sentence[:-1] != '* * * * * * * * * * * * * *': # ignore noise
my_dict = {"sentence": u''.join(sentence[:-1])}
json.dump(my_dict, sentences, ensure_ascii=False)
sentences.write('\n')
break