-
Notifications
You must be signed in to change notification settings - Fork 0
/
mpiexec-ompi4.sh
executable file
·58 lines (44 loc) · 2.07 KB
/
mpiexec-ompi4.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
dir=out.$PSUBMIT_JOBID
rm -rf "$dir"
time1=$(date +"%s")
ALL_ARGS=$(eval echo '' $*)
ALL_ARGS=$(echo $ALL_ARGS | sed "s/%PSUBMIT_JOBID%/$PSUBMIT_JOBID/g")
ALL_ARGS=$(echo $ALL_ARGS | sed "s/%PSUBMIT_NP%/$PSUBMIT_NP/g")
NUL="0"
[ "$PSUBMIT_NP" -gt 10 ] && NUL="00"
[ "$PSUBMIT_NP" -gt 100 ] && NUL="000"
[ "$PSUBMIT_NP" -gt 1000 ] && NUL="0000"
[ "$PSUBMIT_NP" -gt 10000 ] && NUL="00000"
if [ "$ALL_ARGS" == "--show-rank0-out" ]; then
echo "$dir/1/rank.$NUL/stdout"
elif [ "$ALL_ARGS" == "--show-rank0-err" ]; then
echo "$dir/1/rank.$NUL/stderr"
else
[ "$ALL_ARGS" == "--" ] && ALL_ARGS=""
[ -z "$ALL_ARGS" ] || export PSUBMIT_ARGS="$ALL_ARGS"
# The line below is to cut off CUDA from the environment
#LD_LIBRARY_PATH=`echo $LD_LIBRARY_PATH | sed 's!/opt/cuda[^:]*:!:!g'`
[ -z "$NGPUS" ] && NGPUS=0
[ -z "$PSUBMIT_NTH" ] && PSUBMIT_NTH=1
export OMP_NUM_THREADS="$PSUBMIT_NTH"
export PSUBMIT_JOBID PSUBMIT_NP PSUBMIT_NTH PSUBMIT_PPN
[ -z "$PSUBMIT_PREPROC" ] || eval $PSUBMIT_PREPROC
[ -f "hostfile.$PSUBMIT_JOBID" ] && machinefile="-machinefile hostfile.$PSUBMIT_JOBID"
echo ">>> PSUBMIT: mpirun is: " $(which mpirun)
echo ">>> PSUBMIT: mpiexec is: " $(which mpiexec)
echo ">>> PSUBMIT: exetable is: " $(which $TARGET_BIN)
[ -z "$machinefile" ] || prefix="--prefix $(dirname $(dirname $(which mpirun)))"
# echo ">>> PSUBMIT: PATH is: " $PATH
# echo ">>> PSUBMIT: ldd:"
# ldd $(which $TARGET_BIN)
time2=$(date +"%s");
echo $- | grep -q x && omit_setx=true || set -x
mpirun -x OMP_NUM_THREADS -x PATH -x LD_LIBRARY_PATH $prefix $machinefile --bind-to core -np "$PSUBMIT_NP" --map-by ppr:$PSUBMIT_PPN:node --output-filename out.$PSUBMIT_JOBID "$TARGET_BIN" $ALL_ARGS
# for modern ucx-based: add: -mca pml ucx -mca btl ^vader,tcp,openib
[ -z "$omit_setx" ] && set +x
time3=$(date +"%s");
walltime=$(expr $time3 - $time2)
[ "$(expr $time3 - $time1)" -lt "2" ] && sleep $(expr 2 - $time3 + $time1)
echo ">>> PSUBMIT: Walltime: $walltime"
[ -z "$PSUBMIT_POSTPROC" ] || eval $PSUBMIT_POSTPROC
fi