-
Notifications
You must be signed in to change notification settings - Fork 0
/
analyzer.py
150 lines (124 loc) · 4.87 KB
/
analyzer.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
import csv
import cPickle
import datetime
# Extract intra-individual features
def extractFeatures(filename):
answers, questions = filename
def compileUserScores(answers):
scoreDict = {}
with open('data/proc/%s.csv' % answers, 'rb') as csvFile:
csvReader = csv.DictReader(csvFile)
for row in csvReader:
userId = row['OwnerUserId']
score = int(row['Score'])
scoreDict.update({
userId: scoreDict.get(userId, 0) + score
})
return scoreDict
def compileQuestionData(questions, answers):
questionDict = {}
# For each question, get the number of answers
with open('data/proc/%s.csv' % answers, 'rb') as csvFile:
csvReader = csv.DictReader(csvFile)
for row in csvReader:
questionId = row['ParentId']
questionDict.update({
questionId: {
'competitor_num': questionDict.get(questionId, {}).get('competitor_num', 0) + 1,
}
})
# For each question, get the creation date
with open('data/proc/%s.csv' % questions, 'rb') as csvFile:
csvReader = csv.DictReader(csvFile)
for row in csvReader:
questionId = row['Id']
question = questionDict.get(questionId)
if question is None:
continue
else:
question.update({
'CreationDate': row['CreationDate']
})
questionDict.update({
questionId: question
})
return questionDict
print('Extracting features...')
print('1. Compile user data')
scoreDict = compileUserScores(answers)
print('2. Compile question data')
questionDict = compileQuestionData(questions, answers)
print('3. Compute features')
dataPoints = {}
with open('data/proc/%s.csv' % answers, 'rb') as csvFile:
csvReader = csv.DictReader(csvFile)
for row in csvReader:
try:
questionId = row['ParentId']
# Feature 3
url_tag = row['urlCount']
# Feature 4
pic = row['imgCount']
# Feature 5
code = row['codeCount']
# Feature 6
body = row['Body']
ans_len_char = len(body)
ans_len_word = len(body.split())
# Feature 7
paragraphs = body.split('\n')
L = []
sumL = 0
M = 0
for p in paragraphs:
paraLength = len(p)
if paraLength > 0:
L.append(paraLength) # Length of ith paragraph
sumL += paraLength
else:
M -= 1
M += len(paragraphs) # Number of paragraphs
try:
L.sort()
maxL = L.pop()
except:
maxL = 0
readability1 = maxL # Length of longest paragraph
readability2 = float(sumL+1)/float(M+1) # Average length per paragraph
# Feature 8
userId = row['OwnerUserId']
reputation = scoreDict.get(userId, 0)
# Feature 10
FMT = '%Y-%m-%dT%H:%M:%SZ'
questionTime = datetime.datetime.strptime(questionDict[questionId]['CreationDate'], FMT)
answerTime = datetime.datetime.strptime(row['CreationDate'], FMT)
timeSlot = (answerTime - questionTime).seconds
# Feature 14
competitor_num = questionDict[questionId]['competitor_num'] - 1
point = {
'url_tag': url_tag,
'pic': pic,
'code': code,
'ans_len': ans_len_char,
'readability1': readability1,
'readability2': readability2,
'owner_score': reputation,
'timeSlot': timeSlot,
'competitor_num': competitor_num,
}
dataPoints.update({
row['Id']: point,
})
except Exception as e:
print(e)
print('Writing data points in file (pickle)')
with open('data/pickle/features_%s.pkl' % answers, 'wb') as pklFile:
cPickle.dump(dataPoints, pklFile, cPickle.HIGHEST_PROTOCOL)
def main():
filename = [('AnswersPython', 'QuestionsPython'), ('AnswersStack', 'QuestionsStack')]
for name in filename:
print('Analyzing %s and %s...' % (name[0], name[1]))
extractFeatures(name)
print('')
if __name__ == '__main__':
main()