-
Notifications
You must be signed in to change notification settings - Fork 0
/
run.sh
executable file
·131 lines (114 loc) · 2.7 KB
/
run.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
print_help(){
echo "Usage: $0 [option]"
echo "Options:"
echo "create_venv - creates virtual environment called cce"
echo "activate_venv - activates the created virtual environment"
echo "prepare_g4 - prepares the G4 corpus"
echo "prepare_tuba - prepares the TüBa corpus"
echo "prepare_tiger - prepares the TIGER corpus"
echo "train - asks for some inputs and trains a model based on them"
echo "evaluate - evaluate a trained model"
}
check_virtualenv() {
if ! command -v virtualenv &> /dev/null; then
echo "virtualenv is not installed. Installing..."
python3 -m pip install --user virtualenv
echo "virtualenv installation complete."
fi
}
create_venv() {
# Check if virtualenv is installed, if not, install it
check_virtualenv
local env_name="cce"
if [ -d "$env_name" ]; then
echo "Virtual environment '$env_name' already exists. Aborting."
return 1
fi
python3 -m venv "$env_name"
source "./$env_name/bin/activate"
pip install -U pip
echo "Installing some requirements..."
pip install transformers
pip install datasets
pip install tokenizers
pip install pytorch
pip install matplotlib
pip install numpy
pip install tqdm
pip install argparse
pip install nltk
echo "Please run: nltk.download("punkt") once!"
echo "Done!"
}
activate_venv() {
local env_name="cce"
if [ ! -d "$env_name" ]; then
echo "Virtual environment '$env_name' not found. Use '$0 create' to create it."
return 1
fi
source "./$env_name/bin/activate"
}
prepare_g4(){
activate_venv
chmod +x convert_file.sh
python3 download_files.py
mkdir base_files
mv *.jsonl base_files/
mkdir correct_files
python3 write_correct_files.py
rm -r base_files
python3 get_translation_pairs.py
rm -r correct_files
}
prepare_tuba(){
mkdir tmp_json
python3 get_ellipsis_data_tüba.py
rm -r tmp_json
}
prepare_tiger(){
mkdir tmp_json2
python3 get_ellipsis_data_tiger.py
rm -r tmp_json2
}
train(){
cd src
chmod +x train.sh
./train.sh
}
evaluate(){
cd src
chmod +x evaluate.sh
./evaluate.sh
}
if [ "$1" = "--help" ] || [ "$1" = "-h" ]; then
print_help
return 0
fi
case "$1" in
"create_venv")
create_venv
;;
"activate_venv")
activate_venv
;;
"prepare_g4")
prepare_g4
;;
"train")
train
;;
"evaluate")
evaluate
;;
"prepare_tuba")
prepare_tuba
;;
"prepare_tiger")
prepare_tiger
;;
*)
echo "Unknown option: $1"
print_help
exit 1
;;
esac