-
Notifications
You must be signed in to change notification settings - Fork 0
/
psub_lsf.sh
executable file
·83 lines (73 loc) · 2.41 KB
/
psub_lsf.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
#!/bin/bash
. $PSUBMIT_DIRNAME/psub_common.sh
psub_check_job_status() {
queue_out=$(bjobs $jobid_short 2>&1 | grep "^$jobid_short")
if [ -z "$queue_out" -a ! -f "$FILE_OUT" ]; then echo "JOB DISAPPEARED!"; jobstatus="NONE"; return; fi
jobstatus=$(echo "$queue_out" | awk '{print $3}')
if [ "$jobstatus" == "" ]; then echo "JOB DISAPPEARED!"; jobstatus="NONE"; return; fi
case "$jobstatus" in
RUN) jobstatus="R"
;;
PEND) jobstatus="Q"
;;
EXIT) jobstatus="C"
;;
DONE) jobstatus="DONE"
;;
*) echo ">> psub_lsf: UNEXPECTED bjobs output: $queue_out"; jobstatus="NONE";
;;
esac
psub_update_oldjobstatus
}
psub_check_job_done() {
if [ "$jobdone" == "1" ]; then return; fi
psub_check_if_exited
if [ "$?" == 1 ]; then
jobdone=1
psub_check_job_status
local n=1
while [ "$jobstatus" == "R" ]; do
sleep 3
psub_check_job_status
n=$(expr $n \+ 1)
if [ "$n" == "100" ]; then echo ">> psub_lsf: psub_check_job_done: n=$n"; break; fi
done
jobstatus="DONE"
fi
psub_update_oldjobstatus
}
psub_cancel() {
if [ "$jobid" != "" -a "$jobcancelled" == "" ]; then bkill "$jobid"; jobcancelled="$jobid"; fi
}
psub_submit() {
local outfile=`mktemp -u .out.XXXXXXX`
rm -f "$outfile"
local select="select[type==any]"
[ -z "$NODETYPE" ] || select="select[$NODETYPE]";
local span="span[ptile=$PPN]"
n=$(expr "$NNODES" \* "$PPN")
bsub -q "$QUEUE" -l NMIWATCHDOG_OFF=1 -n $n -R "{$select $span}" -W 00:$TIME_LIMIT -o lsf-output.%J psubmit-mpiexec-wrapper.sh -t lsf -i $jobid_short -n "$n" -p "$PPN" -h "$NTH" -g "$NGPUS" -d "$PSUBMIT_DIRNAME" -e "$TARGET_BIN" -o "$OPTSCRIPT" -a "\"$ARGS\"" >& "$outfile"
local pattern="^Job <[0-9]*> is submitted to queue <"
grep -q "$pattern" "$outfile"
if [ $? != "0" ]; then rm -f "$outfile"; exit 0; fi
local submitted=$(grep "$pattern" "$outfile")
jobid=$(echo $submitted | awk -F '[<>]' '{print $2}')
jobid_short=$jobid
rm -f "$outfile"
echo $submitted
}
psub_set_paths() {
psub_common_set_paths
}
psub_set_outfiles() {
FILE_OUT=$SCRATCH_PWD/lsf-output.$jobid
}
psub_move_outfiles() {
psub_common_move_outfiles
}
function psub_make_stackfile() {
psub_common_make_stackfile
}
psub_cleanup() {
psub_common_cleanup
}