-
Notifications
You must be signed in to change notification settings - Fork 8
/
check_kaggle.py
54 lines (45 loc) · 1.51 KB
/
check_kaggle.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
#! /usr/bin/python3
# Purpose: Check kaggle score for HW1
# Author: Roy Lu
# It is an example, you should revise it for your own purpose
import sys
private_simple_baseline = 7.30589
private_strong_baseline = 7.00617
public_simple_baseline = 6.03348
public_strong_baseline = 5.80008
tolerance_rate = 1.2
leaderboard_score = sys.argv[1]
reproduced_score_fn = sys.argv[2]
# The score in HW1 is rmse; therefore smaller is better
hw1_score = 9999
hw1_best_score = 9999
with open(reproduced_score_fn, 'r') as f:
lines = f.readlines()
if len(lines) == 2:
hw1_score = float(lines[0])
hw1_best_score = float(lines[1])
private_score = 9999
public_score = 9999
if len(leaderboard_score.split(',')) == 3:
private_score = float(leaderboard_score.split(',')[1])
public_score = float(leaderboard_score.split(',')[2])
required_score = tolerance_rate * (private_score + public_score) / 2
# 0: reproduced
# 1: public simple baseline
# 2: public strong baseline
# 3: private simple baseline
# 4: private strong baseline
final_score = [0, 0, 0, 0, 0]
if hw1_score <= required_score or hw1_best_score <= required_score:
final_score[0] = 1
# public
if public_score <= public_simple_baseline:
final_score[1] = 1
if public_score <= public_strong_baseline:
final_score[2] = 1
# private
if private_score <= private_simple_baseline:
final_score[3] = 1
if private_score <= private_strong_baseline:
final_score[4] = 1
print(",".join([ str(x) for x in final_score]))