-
Notifications
You must be signed in to change notification settings - Fork 2
/
evaluation.py
207 lines (190 loc) · 7.47 KB
/
evaluation.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
import gc
import multiprocessing
import time
import numpy as np
from utils import *
from scipy.spatial.distance import cdist
from sklearn import preprocessing
from sklearn.metrics.pairwise import euclidean_distances
def test(embeds1, embeds2, top_k, threads_num, metric='inner', normalize=False, csls_k=0, accurate=True):
alignment_rest_12, hits1_12, mr_12, mrr_12 = greedy_alignment(embeds1, embeds2, top_k, threads_num,
metric, normalize, csls_k, accurate)
return alignment_rest_12, hits1_12, mrr_12
def early_stop(flag1, flag2, flag):
if flag <= flag2 <= flag1:
return flag2, flag, True
else:
return flag2, flag, False
def greedy_alignment(logger, embed1, embed2, top_k, nums_threads, metric, normalize, csls_k, accurate):
"""
Parameters
----------
embed1 : matrix_like
An embedding matrix of size n1*d, where n1 is the number of embeddings and d is the dimension.
embed2 : matrix_like
An embedding matrix of size n2*d, where n2 is the number of embeddings and d is the dimension.
top_k : list of integers
Hits@k metrics for evaluating results.
nums_threads : int
The number of threads used to search alignment.
metric : string
The distance metric to use. It can be 'cosine', 'euclidean' or 'inner'.
normalize : bool, true or false.
Whether to normalize the input embeddings.
csls_k : int
K value for csls. If k > 0, enhance the similarity by csls.
Returns
-------
alignment_rest : list, pairs of aligned entities
hits1 : float, hits@1 values for alignment results
mr : float, MR values for alignment results
mrr : float, MRR values for alignment results
"""
t = time.time()
sim_mat = sim(embed1, embed2, metric=metric, normalize=normalize, csls_k=csls_k)
num = sim_mat.shape[0]
if nums_threads > 1:
hits = [0] * len(top_k)
mr, mrr = 0, 0
alignment_rest = set()
rests = list()
search_tasks = task_divide(np.array(range(num)), nums_threads)
pool = multiprocessing.Pool(processes=len(search_tasks))
for task in search_tasks:
mat = sim_mat[task, :]
rests.append(pool.apply_async(calculate_rank, (task, mat, top_k, accurate, num)))
pool.close()
pool.join()
for rest in rests:
sub_mr, sub_mrr, sub_hits, sub_hits1_rest = rest.get()
mr += sub_mr
mrr += sub_mrr
hits += np.array(sub_hits)
alignment_rest |= sub_hits1_rest
else:
mr, mrr, hits, alignment_rest = calculate_rank(list(range(num)), sim_mat, top_k, accurate, num)
assert len(alignment_rest) == num
hits = np.array(hits) / num * 100
for i in range(len(hits)):
hits[i] = round(hits[i], 3)
cost = time.time() - t
if accurate:
if csls_k > 0:
logger.info("accurate results with csls: csls={}, hits@{} = {}%, mr = {:.3f}, mrr = {:.6f}, time = {:.3f} s ".
format(csls_k, top_k, hits, mr, mrr, cost))
else:
logger.info("accurate results: hits@{} = {}%, mr = {:.3f}, mrr = {:.6f}, time = {:.3f} s ".
format(top_k, hits, mr, mrr, cost))
else:
if csls_k > 0:
logger.info("quick results with csls: csls={}, hits@{} = {}%, time = {:.3f} s ".format(csls_k, top_k, hits, cost))
else:
logger.info("quick results: hits@{} = {}%, time = {:.3f} s ".format(top_k, hits, cost))
hits1 = hits[0]
del sim_mat
gc.collect()
return alignment_rest, hits1, mr, mrr
def calculate_rank(idx, sim_mat, top_k, accurate, total_num):
assert 1 in top_k
mr = 0
mrr = 0
hits = [0] * len(top_k)
hits1_rest = set()
for i in range(len(idx)):
gold = idx[i]
if accurate:
rank = (-sim_mat[i, :]).argsort()
else:
rank = np.argpartition(-sim_mat[i, :], np.array(top_k) - 1)
hits1_rest.add((gold, rank[0]))
assert gold in rank
rank_index = np.where(rank == gold)[0][0]
mr += (rank_index + 1)
mrr += 1 / (rank_index + 1)
for j in range(len(top_k)):
if rank_index < top_k[j]:
hits[j] += 1
mr /= total_num
mrr /= total_num
return mr, mrr, hits, hits1_rest
def sim(embed1, embed2, metric='inner', normalize=False, csls_k=0):
"""
Compute pairwise similarity between the two collections of embeddings.
Parameters
----------
embed1 : matrix_like
An embedding matrix of size n1*d, where n1 is the number of embeddings and d is the dimension.
embed2 : matrix_like
An embedding matrix of size n2*d, where n2 is the number of embeddings and d is the dimension.
metric : str, optional, inner default.
The distance metric to use. It can be 'cosine', 'euclidean', 'inner'.
normalize : bool, optional, default false.
Whether to normalize the input embeddings.
csls_k : int, optional, 0 by default.
K value for csls. If k > 0, enhance the similarity by csls.
Returns
-------
sim_mat : An similarity matrix of size n1*n2.
"""
if normalize:
embed1 = preprocessing.normalize(embed1)
embed2 = preprocessing.normalize(embed2)
if metric == 'inner':
sim_mat = np.matmul(embed1, embed2.T) # numpy.ndarray, float32
elif metric == 'cosine' and normalize:
sim_mat = np.matmul(embed1, embed2.T) # numpy.ndarray, float32
elif metric == 'euclidean':
sim_mat = 1 - euclidean_distances(embed1, embed2)
print(type(sim_mat), sim_mat.dtype)
sim_mat = sim_mat.astype(np.float32)
elif metric == 'cosine':
sim_mat = 1 - cdist(embed1, embed2, metric='cosine') # numpy.ndarray, float64
sim_mat = sim_mat.astype(np.float32)
elif metric == 'manhattan':
sim_mat = 1 - cdist(embed1, embed2, metric='cityblock')
sim_mat = sim_mat.astype(np.float32)
else:
sim_mat = 1 - cdist(embed1, embed2, metric=metric)
sim_mat = sim_mat.astype(np.float32)
if csls_k > 0:
sim_mat = csls_sim(sim_mat, csls_k)
return sim_mat
def csls_sim(sim_mat, k):
"""
Compute pairwise csls similarity based on the input similarity matrix.
Parameters
----------
sim_mat : matrix-like
A pairwise similarity matrix.
k : int
The number of nearest neighbors.
Returns
-------
csls_sim_mat : A csls similarity matrix of n1*n2.
"""
nearest_values1 = calculate_nearest_k(sim_mat, k)
nearest_values2 = calculate_nearest_k(sim_mat.T, k)
csls_sim_mat = 2 * sim_mat.T - nearest_values1
csls_sim_mat = csls_sim_mat.T - nearest_values2
return csls_sim_mat
def calculate_nearest_k(sim_mat, k):
sorted_mat = -np.partition(-sim_mat, k + 1, axis=1) # -np.sort(-sim_mat1)
nearest_k = sorted_mat[:, 0:k]
return np.mean(nearest_k, axis=1)
def csls_sim_multi_threads(sim_mat, k, nums_threads):
tasks = task_divide(np.array(range(sim_mat.shape[0])), nums_threads)
pool = multiprocessing.Pool(processes=len(tasks))
rests = list()
for task in tasks:
rests.append(pool.apply_async(calculate_nearest_k, (sim_mat[task, :], k)))
pool.close()
pool.join()
sim_values = None
for res in rests:
val = res.get()
if sim_values is None:
sim_values = val
else:
sim_values = np.append(sim_values, val)
assert sim_values.shape[0] == sim_mat.shape[0]
return sim_values