-
Notifications
You must be signed in to change notification settings - Fork 0
/
mpiexec-impixtmp.sh
executable file
·58 lines (44 loc) · 1.89 KB
/
mpiexec-impixtmp.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
rm -f err.$PSUBMIT_JOBID.* out.$PSUBMIT_JOBID.*
time1=$(date +"%s")
ALL_ARGS=$(eval echo '' $*)
ALL_ARGS=$(echo $ALL_ARGS | sed "s/%PSUBMIT_JOBID%/$PSUBMIT_JOBID/g")
ALL_ARGS=$(echo $ALL_ARGS | sed "s/%PSUBMIT_NP%/$PSUBMIT_NP/g")
if [ "$ALL_ARGS" == "--show-rank0-out" ]; then
echo "out.$PSUBMIT_JOBID.0"
elif [ "$ALL_ARGS" == "--show-rank0-err" ]; then
echo "err.$PSUBMIT_JOBID.0"
else
[ "$ALL_ARGS" == "--" ] && ALL_ARGS=""
[ -z "$ALL_ARGS" ] || export PSUBMIT_ARGS="$ALL_ARGS"
# The line below is to cut off CUDA from the environment
#LD_LIBRARY_PATH=`echo $LD_LIBRARY_PATH | sed 's!/opt/cuda[^:]*:!:!g'`
echo ">>> PSUBMIT: mpiexec is: " $(which mpiexec)
echo ">>> PSUBMIT: Executable is: " $(which $TARGET_BIN)
# echo ">>> PSUBMIT: ldd:"
# ldd $(which $TARGET_BIN)
export PSUBMIT_JOBID PSUBMIT_NP
[ -z "$PSUBMIT_PREPROC" ] || eval $PSUBMIT_PREPROC
[ -f "hostfile.$PSUBMIT_JOBID" ] && machinefile="-machinefile hostfile.$PSUBMIT_JOBID"
[ ! -f $TARGET_BIN ] && TARGET_BIN=$(which $TARGET_BIN)
newexecname=$(mktemp)
nds=$(echo $PSUBMIT_NODELIST | sed 's/,/ /g')
for i in $nds; do
node=$(echo $i | cut -d: -f1);
scp "$TARGET_BIN" ${node}:$newexecname;
ssh ${node} "chmod +x $newexecname";
done
time2=$(date +"%s");
export I_MPI_HYDRA_BOOTSTRAP="ssh"
echo $- | grep -q x && omit_setx=true || set -x
mpiexec.hydra $machinefile -np "$PSUBMIT_NP" -ppn "$PSUBMIT_PPN" --errfile-pattern=err.$PSUBMIT_JOBID.%r --outfile-pattern=out.$PSUBMIT_JOBID.%r "$newexecname" $ALL_ARGS
[ -z "$omit_setx" ] && set +x
for i in $nds; do
node=$(echo $i | cut -d: -f1);
ssh $node "rm -f $newexecname";
done
time3=$(date +"%s");
walltime="$(expr $time3 - $time2)"
[ "$(expr $time3 - $time1)" -lt "2" ] && sleep $(expr 2 - $time3 + $time1)
echo ">>> PSUBMIT: Walltime: $walltime"
[ -z "$PSUBMIT_POSTPROC" ] || eval $PSUBMIT_POSTPROC
fi