-
Notifications
You must be signed in to change notification settings - Fork 0
/
psub_common.sh
executable file
·143 lines (137 loc) · 5.54 KB
/
psub_common.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
function psub_check_if_exited() {
cat "$FILE_OUT" | awk -v NNODES=1 '/Exiting.../ { done++; if (done == NNODES) { exit 1; } } { next; }'
}
function psub_update_oldjobstatus() {
if [ "$jobstatus" != "$oldjobstatus" ]; then echo "Job status: $jobstatus"; fi
oldjobstatus=$jobstatus
}
function psub_common_set_paths() {
SCRATCH_PWD="$PWD"
SCRATCH_HOME="$HOME"
}
function psub_common_move_outfiles() {
[ ! -f "$FILE_OUT" -o "$jobid" == "" ] && return
local dir="$SCRATCH_PWD"
local home="$SCRATCH_HOME"
mkdir -p $dir/results.$jobid_short
if [ -f $FILE_OUT ]; then
mv $FILE_OUT $dir/results.$jobid_short
oldpwd=$(pwd)
cd $dir/
ln -s $(basename $FILE_OUT) batch.${jobid_short}.out
cd $oldpwd
fi
[ -f psubmit_wrapper_output.$jobid_short ] && mv psubmit_wrapper_output.$jobid_short $dir/results.$jobid_short
[ -f hostfile.$jobid_short ] && mv hostfile.$jobid_short $dir/results.$jobid_short
local rank0=""
local erank0=""
local errfiles=""
local r=$(ls -1d $dir/out.$jobid_short.* 2> /dev/null)
if [ "$r" != "" ]; then
mv $dir/out.${jobid_short}.* $dir/results.$jobid_short
PSUBMIT_NP=$(expr $NNODES \* $PPN)
PSUBMIT_JOBID=$jobid_short
local f=$(. "$MPIEXEC" --show-rank0-out)
if [ "$f" != "" ]; then
if [ -f "$dir/results.$jobid_short/$f" ]; then
olddir=$PWD
cd "$dir/results.$jobid_short"
ln -s "$f" "rank0"
rank0="TRUE"
cd $olddir
fi
fi
fi
r=$(ls -1d $dir/err.$jobid_short.* 2> /dev/null)
if [ "$r" != "" ]; then
mv $dir/err.$jobid_short.* $dir/results.$jobid_short
PSUBMIT_NP=$(expr $NNODES \* $PPN)
PSUBMIT_JOBID=$jobid_short
f=$(. "$MPIEXEC" --show-rank0-err)
if [ "$f" != "" ]; then
if [ -f "$dir/results.$jobid_short/$f" ]; then
olddir=$PWD
cd "$dir/results.$jobid_short"
ln -s "$f" "erank0"
erank0="TRUE"
cd $olddir
fi
fi
errfiles="TRUE"
fi
r=$(ls -1d $dir/*.${jobid_short}.* $dir/*.${jobid_short} 2> /dev/null)
if [ "$r" != "" ]; then
for f in $r; do
x=$(basename $f | grep '^[^.]*\.'${jobid_short}'\.[^.]*$')
[ -z "$x" ] || mv "$f" $dir/results.$jobid_short
x=$(basename $f | grep '^.*\.'${jobid_short}'$')
if [ "$x" != "results.$jobid_short" ]; then
[ -z "$x" ] || mv "$f" $dir/results.$jobid_short
fi
done
fi
echo "Results collected:" "results.${jobid_short}/"
[ -z "$rank0" ] || echo "Rank 0 output:" results.$jobid_short/rank0
[ -z "$erank0" ] || echo "Rank 0 errout:" results.$jobid_short/erank0
echo "Batch system output:" results.$jobid_short/`basename $FILE_OUT`
echo "Psubmit wrapper output:" results.$jobid_short/psubmit_wrapper_output.$jobid_short
if true; then
echo -ne "\n--- Batch system output: ---\n"
tail -n15 $dir/results.$jobid_short/$(basename $FILE_OUT)
echo -ne "\n--- Psubmit wrapper output: ---\n"
tail -n15 $dir/results.$jobid_short/psubmit_wrapper_output.$jobid_short
[ -z "$rank0" ] || ( echo -ne "\n--- Rank 0 output: ---\n" && tail -n15 $dir/results.$jobid_short/rank0 )
[ -z "$erank0" ] || ( echo -ne "\n--- Rank 0 errout: ---\n" && tail -n15 $dir/results.$jobid_short/erank0 )
[ -z "$errfiles" ] || ( echo -ne "\n--- NOTE: THERE ARE ERROR OUTPUT FILES\n" && cat $dir/results.$jobid_short/err.$jobid_short.* | head )
fi
}
function psub_common_make_stackfile() {
local dir="$SCRATCH_PWD"
local stackdir=$dir/results.$jobid_short
if [ -d "$dir/results.$jobid_short/out.$jobid_short" ]; then
stackdir="$dir/results.$jobid_short"
fi
rm -f __result.$jobid_short __stack.$jobid_short
touch __stack.$jobid_short
rm -f stacktrace.$jobid_short
for f in $(find "$stackdir" -type f); do
local ST
egrep "(: Assertion .* failed.)" $f >> __result.$jobid_short && ST=1
if [ -z "$ST" ]; then
egrep "(>> TIMEOUT)|(>> FATAL SIGNAL: [0-9]*)|(>> EXCEPTION)" $f >> __result.$jobid_short && ST=1;
fi
if [ ! -z "$ST" ]; then
if grep -q 'Stack trace:' $f ; then
echo "File: $f" >> __stack.$jobid_short
echo "" >> __stack.$jobid_short
cat $f | awk '/Stack trace:/{STAT=1} /-------------------------------------------/{ if (STAT==2) STAT=3; if (STAT==1) STAT=2; } STAT>0 { print; if (STAT==3) STAT=0 }' >> __stack.$jobid_short
echo "" >> __stack.$jobid_short
fi
fi
done
local lines=$(wc -l __result.$jobid_short | cut -d' ' -f1)
if [ "$lines" != "0" ]; then
local R=$(sort -u < __result.$jobid_short | head -n1)
case "$R" in
*">> T"*) echo -ne ">> STATUS: TIMEOUT\n\n" >> stacktrace.$jobid_short;;
*">> E"*) echo -ne ">> STATUS: EXCEPTION\n\n" >> stacktrace.$jobid_short;;
*">> F"*) echo -ne ">> STATUS: CRASH\n\n" >> stacktrace.$jobid_short;;
*": As"*) echo -ne ">> STATUS: ASSERT\n\n" >> stacktrace.$jobid_short;;
*) echo -ne "STATUS: ???\n\n" >> stacktrace.$jobid_short;;
esac
echo "$R" >> stacktrace.$jobid_short
echo >> stacktrace.$jobid_short
cat __stack.$jobid_short >> stacktrace.$jobid_short
mv stacktrace.$jobid_short $dir/results.$jobid_short
fi
rm -f __result.$jobid_short __stack.$jobid_short
}
function psub_common_cleanup() {
echo "CLEANUP..."
psub_cancel
sleep 1
psub_move_outfiles
rm -f "$OUTFILE"
echo "CLEANUP DONE"
exit 1
}