-
Notifications
You must be signed in to change notification settings - Fork 0
/
psub_vbbs.sh
executable file
·103 lines (87 loc) · 3.4 KB
/
psub_vbbs.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
#!/bin/bash
. $PSUBMIT_DIRNAME/psub_common.sh
vbbs_id=0
outfile=""
function psub_check_job_status() {
if [ "$jobstatus" == "E" ]; then jobstatus="DONE"; return; fi
if [ "$jobstatus" == "DONE" ]; then return; fi
if [ "$jobstatus" == "Q" ]; then
NBGJOBS=$(jobs -rp | wc -l)
if [ "$NBGJOBS" -eq "0" ]; then
jobstatus="R"
# make up a node list
cat $outfile | grep 'node:' | awk -vppn=$PPN '{print $2 ":" ppn }' > hostfile.$jobid
for n in $(cat hostfile.$jobid); do
PSUBMIT_NODELIST=$PSUBMIT_NODELIST,${n}
done
export PSUBMIT_NODELIST=`echo $PSUBMIT_NODELIST | sed 's/^,//'`
export SLURM_JOBID=$(vbbs show_slurm_id 1 | grep SLURM_JOBID | cut -f2 -d' ')
if [ ${SLURM_JOBID} -lt 1 ]; then
unset SLURM_JOBID
fi
# ssh and start run-mpi script
# headnode=$(head -n1 hostfile.$jobid | cut -d: -f1)
NP=$(expr "$NNODES" \* "$PPN")
cat $outfile > $FILE_OUT
# Start a background mpiexec
timeout -k20 ${TIME_LIMIT}m $PSUBMIT_DIRNAME/psubmit-mpiexec-wrapper.sh -t vbbs -i $jobid_short -n "$NP" -p "$PPN" -h "$NTH" -g "$NGPUS" -d "$PSUBMIT_DIRNAME" -e "$TARGET_BIN" -o "$OPTSCRIPT" -a "\"$ARGS\"" > "$FILE_OUT" 2>&1 &
# Start a background ssh session
#ssh $headnode export PATH=$PATH \&\& export LD_LIBRARY_PATH="$LD_LIBRARY_PATH" \&\& export PSUBMIT_NODELIST="$PSUBMIT_NODELIST" \&\& cd "$PWD" \&\& timeout -k20 ${TIME_LIMIT}m $PSUBMIT_DIRNAME/psubmit-mpiexec-wrapper.sh -t vbbs -i $jobid_short -n "$NP" -p "$PPN" -d "$PSUBMIT_DIRNAME" -o "$OPTSCRIPT" -a "\"$ARGS\"" > "$FILE_OUT" 2>&1 &
fi
fi
if [ "$jobstatus" == "R" ]; then
NBGJOBS=$(jobs -rp | wc -l)
if [ "$NBGJOBS" -eq "0" ]; then
jobstatus="DONE"
vbbs stop $jobid
#rm -f hostfile.$jobid
fi
fi
psub_update_oldjobstatus
}
function psub_check_job_done() {
[ "$jobstatus" == "DONE" ] && jobdone=1
true;
}
function psub_cancel() {
if [ "$jobid" != "" -a "$jobcancelled" == "" ]; then
# ssh to a node and kill
# headnode=$(head -n1 hostfile.$jobid | cut -d: -f1)
# ssh $headnode killall psubmit-mpiexec-wrapper.sh
killall psubmit-mpiexec-wrapper.sh
vbbs stop $jobid
jobcancelled="$jobid"
fi
}
function psub_submit() {
outfile=$(mktemp -u .out.XXXXXXX)
rm -f "$outfile"
NBGJOBS=$(jobs -rp | wc -l)
if [ "$NBGJOBS" -ne "0" ]; then echo ">> psub_vbbs: ERROR: non-zero background jobs count: $NBGJOBS"; exit 1; fi
vbbs start $NNODES >& $outfile &
if [ $? != 0 ]; then echo ">> psub_vbbs: VBBS failed on start"; exit 1; fi
jobstatus="Q"
local cnt=0
while [ "$cnt" -lt 60 ]; do
jobid=$(cat $outfile | grep 'id:' | cut -d ' ' -f2)
if [ -z "$jobid" ]; then sleep 1; cnt=$(expr "$cnt" \+ 1); continue; fi
jobid_short=$jobid
break;
done
if [ -z "$jobid" ]; then jobstatus="NONE"; echo ">> psub_vbbs: cannot find output of 'vbbs start'"; return; fi
}
function psub_set_paths() {
psub_common_set_paths
}
function psub_set_outfiles() {
FILE_OUT=$SCRATCH_PWD/vbbs-$jobid.out
}
function psub_move_outfiles() {
psub_common_move_outfiles
}
function psub_make_stackfile() {
psub_common_make_stackfile
}
function psub_cleanup() {
psub_common_cleanup
}