mkcloud

#!/bin/bash
#
# mkcloud - Setup a virtual cloud on one system (physical or even virtual)
#
# Authors: J. Daniel Schmidt <jdsn@suse.de>
#          Bernhard M. Wiedemann <bwiedemann@suse.de>
#
# 2012, SUSE LINUX Products GmbH
#

# Quick introduction:
#
# 1. read the usage: mkcloud help
#    or visit https://github.com/SUSE-Cloud/automation/blob/master/docs/mkcloud.md
# 2. This tool relies on the script qa_crowbarsetup.sh
# 3. Please 'export' environment variables according to your needs.

: ${cloud:=cloud}
log_dir_default=/var/log/mkcloud/$cloud
[[ $UID != 0 ]] && log_dir_default="./log"
: ${log_dir:=$log_dir_default}
mkdir -p "$log_dir"
log_file=$log_dir/`date -Iseconds`.log
exec >  >(exec tee -ia $log_file)
exec 2> >(exec tee -ia $log_file >&2)

if [[ $debug_mkcloud = 1 ]] ; then
    set -x
    PS4='+(${BASH_SOURCE##*/}:${LINENO}) ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
fi

cache_dir_default=/var/cache/mkcloud/$cloud
[[ $UID != 0 ]] && cache_dir_default="${XDG_CACHE_HOME:-./cache}/mkcloud"
: ${cache_dir:=$cache_dir_default}
mkdir -p "$cache_dir"

# FIXME: separate user-tweakable parameters from script local variables.
# Currently there is no clearly defined interface point.  One of the
# causes of this is violation of the common shell coding standard which
# uses uppercase for environment variables and constants, and lowercase
# for local variables.
: ${SCRIPTS_DIR:=$mkcloud_dir} # for backward compatibility
: ${SCRIPTS_DIR:=$(dirname $(readlink -e $0))}
: ${qa_crowbarsetup:=${SCRIPTS_DIR}/qa_crowbarsetup.sh}
: ${iscsictl:=${SCRIPTS_DIR}/iscsictl.py}

: ${mkclouddriver:=libvirt}
scripts_lib_dir=${SCRIPTS_DIR}/lib
# include separate bash libs
# NOTE that this is a temporary solution during refactoring of mkcloud
common_scripts="mkcloud-onhost.sh mkcloud-common.sh mkcloud-driver-$mkclouddriver.sh qa_crowbarsetup-help.sh mkcloud-ipmi.sh"
for script in $common_scripts; do
    source ${scripts_lib_dir}/$script
done
mkcconf=mkcloud.config
rm -f $mkcconf # we dont want to source old info in the line below

: ${virtualcloud:=virtual}
if (( $want_ipv6 > 0 )); then
    echo ">>> WARNING: IPv6 (want_ipv6) is under active development <<<"
    : ${net_admin:='fd00:0:0:3'}
    firmware_type="uefi"
else
    : ${net_admin:=192.168.124}
fi
setcloudnetvars $virtualcloud
: ${forwardmode:=nat}
: ${nodenumber:=2}
# expect to have this many physical machines attached via $mkch_physcloudif
# these replace virtual machines, so we start less VMs
: ${nodenumberphys:=0}
[[ $nodenumberphys = 0 ]] || [[ $mkch_physcloudif ]] || complain 100 "need to set mkch_physcloudif for physical nodes"
# configuration of clusters
: ${clusterconfig:=''}
# '+'-separated list of MAC#serial_of_drbd_volume of the drbd cluster nodes
# (used only internally to transport this information to qa_crowbarsetup):
: ${drbdnode_mac_vol:=''}
: ${cephvolumenumber:=1}
: ${controller_raid_volumes:=0}
: ${vcpus:=2}
nicsdefault=1
[[ $want_ironic ]] && nicsdefault=2
[[ $crowbar_networkingmode = team ]] && nicsdefault=$((nicsdefault + 1))
[[ $want_separate_ceph_network ]] && crowbar_networkingmode="team"
[[ $want_separate_ceph_network ]] && nicsdefault=4
: ${nics:=$nicsdefault}
: ${adminvcpus:=2}
cpuflags=''
working_dir_orig=`pwd`
: ${artifacts_dir:=$working_dir_orig/.artifacts}
mkdir -p $artifacts_dir
start_time=`date`
: ${cloudvg:=cloud}
needcvol=1
: ${vdisk_dir:=/dev/$cloudvg}
: ${admin_node_disk:=$vdisk_dir/$cloud.admin}
: ${admin_node_memory:=4194304}
iscloudver 7plus && : ${controller_node_memory:=12582912}
: ${controller_node_memory:=6291456}
: ${compute_node_memory:=2621440}
[[ "$libvirt_type" = "hyperv" ]] && \
    compute_node_memory=$(max $compute_node_memory ${hyperv_node_memory:-3000000})
[[ "$libvirt_type" = "xen" ]] && \
    compute_node_memory=$(max $compute_node_memory ${xen_node_memory:-4000000})
# hdd size defaults (unless defined otherwise)
: ${adminnode_hdd_size:=15}
[[ "$upgrade_cloudsource" ]] && \
    adminnode_hdd_size=$(max $adminnode_hdd_size 20)
: ${controller_hdd_size:=20}
: ${computenode_hdd_size:=20}
: ${cephvolume_hdd_size:=21}
: ${controller_ceph_hdd_size:=25}
: ${lonelynode_hdd_size:=20}
: ${ironicnode_hdd_size:=20}
: ${want_pci_passthrough:=0}
if [[ $want_pci_passthrough = 1 ]]; then
    # Create extra volume to test pci passthrough
    : ${extravolumenumber:=1}
    : ${extravolume_hdd_size:=1}
fi

if [[ $hacloud = 1 ]] && ( [[ "$want_database_sql_engine" == "postgresql" ]] || iscloudver 6minus ) ; then
    : ${drbd_hdd_size:=15}
else
    : ${drbd_hdd_size:=0}
fi
: ${drbd_database_size:=5}
: ${drbd_rabbitmq_size:=5}
# magnum sets up two nodes using docker/kubernetes on compute node
# each node requiring 2GB of RAM and 20GB of harddisk for the images
if [[ $want_magnum_proposal = 1 ]]; then
    : ${compute_node_memory:=6621440}
    : ${computenode_hdd_size:=80}
fi

# pvlist is filled below
: ${pvlist:=}
next_pv_device=
pv_cur_device_no=0

: ${cloudbr:=${cloud}br}
# IFNAMSIZ - 1 - 4 (vlan suffix likely length)
cloudbrlimit=11
[[ ${#cloudbr} -gt $cloudbrlimit ]] && complain 100 "cloudbr name is too long (> $cloudbrlimit); \$cloud must be shorter"

: ${ironicbr:=${cloud}irobr}
if [[ $want_ironic ]]; then
    [[ ${#ironicbr} -gt $cloudbrlimit ]] && complain 100 "ironicbr name is too long (> $cloudbrlimit); \$cloud must be shorter"

    # network card index for ironic network
    : ${ironicnic:=1}
fi

#if localreposdir_src string is available, the local repositories are used for setup
: ${localreposdir_src:=""}
: ${cct_tests:="features:base"}

[ -n "$cache_clouddata" ] && {
    # Reduce scope a bit for road-warriers
    # Only host architecture, nothing else
    export architectures="$arch"
    # Remove cct: requires SDK to mirror which is huge
    export cct_tests=""
    export localreposdir_src=$cache_dir
    # make sure the cached images are used
    export want_cached_images=1
}

#localreposdir_target is the 9p target dir and also the mount target dir in the VM
: ${localreposdir_target:="/repositories"}
[ -z "$localreposdir_src" ] && localreposdir_target=""
: ${install_chef_suse_override:=./install-chef-suse.sh}
host_mtu=$(determine_mtu)
iscloudver 7plus && [[ $arch == x86_64 ]] && \
    [[ -n $want_efi ]] && firmware_type="uefi"

# TODO(colleen) Fix cloudsource check once there are new MUs for cloud7/8
iscloudver 7plus && [[ $cloudsource =~ develcloud ]] && [[ -z $want_ssl_trusted ]] && {
    export want_ssl_trusted=1
}

pidfile=mkcloud.pid

exec </dev/null

function show_environment
{
    end_time=`date`
    echo
    echo "Crowbar /etc/motd"
    echo "---------------------"
    nc -z -w 1 $adminip 22 && $ssh root@$adminip "cat /etc/motd"
    echo
    echo "Environment Details"
    echo "-------------------------------"
    if $(which git >&/dev/null); then
        pushd $SCRIPTS_DIR
        echo "         git: $(git --no-pager show -s --oneline)"
        popd
    fi
    echo "    hostname: `hostname -f`"
    echo "     started: $start_time"
    echo "       ended: $end_time"
    echo "-------------------------------"
    echo " cloudsource: $cloudsource"
    echo "    TESTHEAD: $TESTHEAD"
    echo " want_test_updates: $want_test_updates"
    echo "    scenario: $scenario"
    echo "  nodenumber: $nodenumber"
    echo "     cloudpv: $cloudpv"
    echo " UPDATEREPOS: $UPDATEREPOS"
    echo " UPDATEREPOS_EXTRA: $UPDATEREPOS_EXTRA"
    echo "    cephvolumenumber: $cephvolumenumber"
    echo " upgrade_cloudsource: $upgrade_cloudsource"
    echo "-------------------------------"
    env | grep -i "^want_"
    echo "-------------------------------"
    if [[ $CI_RUN = 1 ]]; then
        column -s"," -t $timing_file_host  | tee ${timing_file_host}.byTime.txt
        sort -t, -k3,3 -k1,1 $timing_file_host | column -s"," -t  | tee ${timing_file_host}.byType.txt
    fi
}

function pre_exit_cleanup
{
    [[ $want_ironic = 1 ]] && teardown_ironic_test_env
    rm $pidfile
}

function onhost_supportconfig
{
    if [ -z "$SKIPSUPPORTCONFIG" ] ; then
        ssh $sshopts root@$adminip '
            set -x
            for node in $(crowbar machines list | grep ^d) ; do
            (
                echo "Collecting supportconfig from $node"
                timeout 1200 ssh $node supportconfig | wc
                timeout 300 scp $node:/var/log/\*txz /var/log/
            )&
            done
            timeout 600 supportconfig | wc &
            wait
        '
        $scp root@$(wrap_ip $adminip):/var/log/*txz $artifacts_dir/
    fi
}

function onhost_stop_chef_clients_on_nodes
{
    # stop chef-client on nodes so the env will not be altered after a failure
    ssh $sshopts root@$adminip '
            set -x
            for node in $(crowbar machines list | grep ^d) ; do
            (
                echo "Stopping chef-client on $node"
                timeout 30 ssh $node rcchef-client stop
            )
            done
        '
}

function onhost_start_chef_clients_on_nodes
{
    # Reverse effects of onhost_stop_chef_clients_on_nodes
    ssh $sshopts root@$adminip '
            set -x
            for node in $(crowbar machines list | grep ^d) ; do
            (
                echo "Starting chef-client on $node"
                timeout 30 ssh $node rcchef-client start
            )
            done
        '
}

function error_exit
{
    exitcode=$1
    message=$2
    if [[ -n $BUILD_NUMBER ]]; then
        onhost_supportconfig
    fi
    onhost_stop_chef_clients_on_nodes
    pre_exit_cleanup
    echo $message
    show_environment
    echo "exitcode: $exitcode"
    exit $exitcode
} >&2


function sshrun
{
    cat > $mkcconf <<EOF
        export drbdnode_mac_vol=$drbdnode_mac_vol ;
        export cloud=$virtualcloud ;
        # hostnames of repo, nfs, rsync and smt servers
        # (*_ip and *_fqdn suffix variables are computed automatically)
        export reposerver=$reposerver ;
        export reposerver_ip=$reposerver_ip ;
        export reposerver_fqdn=$reposerver_fqdn ;
        export reposerver_base_path=$reposerver_base_path ;
        export reposerver_url=$reposerver_url ;
        export imageserver_url=$imageserver_url ;
        export nfsserver=$nfsserver ;
        export nfsserver_ip=$nfsserver_ip ;
        export nfsserver_fqdn=$nfsserver_fqdn ;
        export nfsserver_base_path=$nfsserver_base_path ;
        export rsyncserver=$rsyncserver ;
        export rsyncserver_ip=$rsyncserver_ip ;
        export rsyncserver_fqdn=$rsyncserver_fqdn ;
        export rsyncserver_images_dir=$rsyncserver_images_dir ;
        export susedownload=$susedownload ;
        export cloudfqdn=$cloudfqdn ;
        export cloudsource=$cloudsource ;
        export mkclouddriver=$mkclouddriver ;
        export adminip=$adminip ;
        export hacloud=$hacloud ;
        export libvirt_type=$libvirt_type ;
        export firmware_type=$firmware_type ;
        export networkingplugin=$networkingplugin ;
        export networkingmode=$networkingmode ;
        export nosetestparameters=${nosetestparameters} ;
        export tempestoptions='${tempestoptions}' ;
        export cephvolumenumber=$cephvolumenumber ;
        export controller_raid_volumes=$controller_raid_volumes ;
        export drbd_database_size=$drbd_database_size ;
        export drbd_rabbitmq_size=$drbd_rabbitmq_size ;
        export shell=$shell ;
        export keep_existing_hostname=$keep_existing_hostname ;
        export http_proxy=$http_proxy ;
        export https_proxy=$https_proxy ;
        export no_proxy=$no_proxy ;
        export cct_tests=$cct_tests ;
        export scenario=$scenario ;
        export host_mtu=$host_mtu ;
        export architectures='$architectures';

        export crowbar_networkingmode=$crowbar_networkingmode;

        export nova_shared_instance_storage=$nova_shared_instance_storage ;

        export localreposdir_target=$localreposdir_target ;

        export cinder_backend=$cinder_backend;
        export cinder_netapp_storage_protocol=$cinder_netapp_storage_protocol;
        export cinder_netapp_login=$cinder_netapp_login;
        export cinder_netapp_password=$cinder_netapp_password;

        export UPDATEREPOS=$UPDATEREPOS ;
        export UPDATEREPOS_UPGRADE=$UPDATEREPOS_UPGRADE ;
        export UPDATEREPOS_EXTRA=$UPDATEREPOS_EXTRA ;
        export TESTHEAD=$TESTHEAD ;
        export NOINSTALLCLOUDPATTERN=$NOINSTALLCLOUDPATTERN ;

        export clouddescription='$clouddescription';
        export JENKINS_BUILD_URL=$BUILD_URL;
        export JENKINS_NODE_NAME=$NODE_NAME;
        export JENKINS_EXECUTOR_NUMBER=$EXECUTOR_NUMBER;
        export JENKINS_WORKSPACE=$WORKSPACE;
        export test_internet_url=$test_internet_url;
        export extraipmipw=$extraipmipw;
        export ipmi_ip_addrs=$ipmi_ip_addrs;
        export bmc_user=$bmc_user;
        export bmc_password=$bmc_password;
        export upgrade_progress_file=$upgrade_progress_file;
        export upgrade_test_stack_count=$upgrade_test_stack_count;
EOF
    # setting these variables within mkcloud does not make them part of the env, so we need to export them
    export nodenumber nodenumberlonelynode nodenumberironicnode clusterconfig ironicnetmask
    env|grep -e ^debug_ -e ^pre_ -e ^vlan_ -e ^want_ -e ^net_ -e ^nodenumber -e ^clusterconfig -e ^ironicnetmask | sort >> $mkcconf

    cp -a $mkcconf $artifacts_dir/
    $scp -r $SCRIPTS_DIR $mkcconf root@$(wrap_ip $adminip):
    [[ $need_scenario = 1 ]] && $scp $scenario_path root@$(wrap_ip $adminip):
    ssh $sshopts root@$adminip "echo `hostname` > cloud ; . ./$(basename $SCRIPTS_DIR)/qa_crowbarsetup.sh ; $@"
    return $?
}

function onadmin
{
    # functions can have parameters, so pass on all except $1
    local cmd=$1
    shift
    safely sshrun "TIMEFORMAT=\"timing for mkcloud call 'onadmin_$cmd' real=%R user=%U system=%S\" ;" \
        time onadmin_$cmd "$@"
    local ret=$?
    # append timing information of admin node to host timing file
    $ssh root@$adminip "( [ -e $timing_file_admin ] && cat $timing_file_admin && rm -f $timing_file_admin ) || :" >> $timing_file_host
    return $ret
}

function onhost
{
    # functions can have parameters, so pass on all except $1
    local cmd=$1
    shift
    safely onhost_$cmd "$@"
}

function cleanup
{
    ipmi_cleanup
    ${mkclouddriver}_do_cleanup
}

function onhost_cleanup_admin_node
{
    ${mkclouddriver}_do_cleanup_admin_node
}

function onhost_get_next_pv_device
{
    ${mkclouddriver}_do_get_next_pv_device
}

function onhost_add_etchosts_entries
{
    grep -q crowbar /etc/hosts || echo "$adminip crowbar.$cloudfqdn crowbar" >> /etc/hosts
}

function setuphost
{
    ${mkclouddriver}_do_setuphost
}

function prepare
{
    if ! [ -e ~/.ssh/id_rsa ]; then
        echo "Creating key for controlling our VMs..."
        ssh-keygen -t rsa -f ~/.ssh/id_rsa -N ""
    fi

    onhost_cacheclouddata
    ${mkclouddriver}_do_prepare
}

function wait_for_crowbar_ssh
{
    wait_for 240 1 "nc -w 1 -z $adminip 22" 'admin node to start ssh daemon'
}

function wait_for_crowbar_ntpd
{
    wait_for 200 10 "ntpdate -d $adminip" "admin node ntpd service"
}

# bring up the VM for crowbar
function setupadmin
{
    ${mkclouddriver}_do_setupadmin

    ping_cmd="ping"
    if (( $want_ipv6 > 0 )); then
        ping_cmd="ping6"
    fi
    wait_for 300 1 "$ping_cmd -q -c 1 -w 1 $adminip >/dev/null" 'crowbar admin VM'
    wait_for_crowbar_ssh

    echo "Injecting public key into admin node..."
    local x keyfile
    for keyfile in ~/.ssh/id_{ed25519,rsa,dsa}.pub ; do
        if ! [ -r $keyfile ]; then
            continue
        fi
        local pubkey=$(< $keyfile)
        ssh_password root@$adminip "
            mkdir -p ~/.ssh;
            grep -q '$pubkey' ~/.ssh/authorized_keys 2>/dev/null ||
                echo '$pubkey' >> ~/.ssh/authorized_keys
        "
        break
    done
    if [[ $user_keyfile ]]; then
        cat $user_keyfile | sshrun "cat >> ~/.ssh/authorized_keys"
    fi
    onadmin write_cloud_info
    echo "you can now proceed with installing crowbar"
    # prevent jumbo frames from going out
    if [[ $want_mtu_size -gt 1500 ]] || [[ $host_mtu -lt 1500 ]]; then
        # we subtract 40 to account for the IP + TCP headers.
        local mss=$(( $host_mtu - 40 ))
        local rule
        # Remove all previous TCPMSS rules
        iptables -S FORWARD | grep -o FORWARD.*TCPMSS.* | while read rule ; do
            $sudo iptables -w -D ${rule} 2>/dev/null
        done
        $sudo iptables -w -I FORWARD -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss $mss
    fi
}

function wait_for_node_shutdown
{
    wait_for 150 1 "LC_ALL=C virsh domstate $1 | grep shut.off" "$1 node to shut down"
}

function remove_snapshot_volume
{
    if $($sudo lvs "$1.snap" > /dev/null 2>&1) ; then
        safely $sudo lvremove -f "$1.snap"
    fi
}

function merge_snapshot_volume
{
    if $($sudo lvs "$1.snap" > /dev/null 2>&1) ; then
        safely $sudo lvconvert --merge "$1.snap"
    fi
}

function create_snapshot_volume
{
    safely $sudo lvcreate -l100%ORIGIN -s -n "$1.snap" "$1"
}

function createadminsnapshot
{
    $sudo virsh shutdown $cloud-admin
    wait_for_node_shutdown $cloud-admin
    remove_snapshot_volume $admin_node_disk
    create_snapshot_volume $admin_node_disk
    $sudo virsh start $cloud-admin
    wait_for_crowbar_ssh
}

function restoreadminfromsnapshot
{
    $sudo virsh destroy $cloud-admin
    merge_snapshot_volume $admin_node_disk
    createadminsnapshot
}

function lvs_lv_name
{
    $sudo lvs -o lv_name ${cloudvg}
}

function createcloudsnapshot
{
    shutdowncloud
    wait_for_node_shutdown $cloud-admin
    local i d
    for i in $(nodes ids all) ; do
        wait_for_node_shutdown $cloud-node$i
    done
    for d in $(lvs_lv_name | grep "${cloud}\..*snap") ; do
        remove_snapshot_volume "${cloudvg}/${d%.snap}"
    done
    for d in $(lvs_lv_name | grep ${cloud}\.) ; do
        create_snapshot_volume "${cloudvg}/${d}"
    done
    restartcloud
}

function restorecloudfromsnapshot
{
    local i d can_restore=1
    lvs_lv_name | grep "${cloud}.admin.snap" || {
        echo "Missing snapshot for ${cloud}.admin"
        can_restore=
    }
    for i in $(nodes ids all) ; do
        if [[ ! ${can_restore} ]]; then
            break
        fi
        lvs_lv_name | grep "${cloud}.node$i.snap" || {
            echo "Missing snapshot for ${cloud}.node$i"
            can_restore=
        }
    done

    if [[ ! ${can_restore} ]]; then
        complain 88 "Cannot restore cloud, missing snapshot(s)"
        return
    fi

    $sudo virsh destroy $cloud-admin
    for i in $(nodes ids all) ; do
        $sudo virsh destroy $cloud-node$i
    done
    for d in $(lvs_lv_name | grep "${cloud}\..*snap") ; do
        merge_snapshot_volume "${cloudvg}/${d%.snap}"
    done
    createcloudsnapshot
}

# bring up lonely_node(s) in the admin network
function setuplonelynodes
{
    ${mkclouddriver}_do_setuplonelynodes
}

# bring up ironic nodes in the ironic network
function setupironicnodes
{
    if [[ $(nodes number ironic) -lt 1 ]] ; then
        complain 80 "Please set nodenumberironicnode."
        return
    fi

    ${mkclouddriver}_do_setupironicnodes
}

# register lonely_node against crowbar
function crowbar_register
{
    local i
    for i in $(nodes ids lonely) ; do
        local mac=$(macfunc $i)
        sshrun "lonelymac=$mac adminip=$adminip onadmin_crowbar_register" &
    done
    wait
}

# setup an NFS server on the first lonely node
function lonelynode_nfs_server
{
    local nfsserver=($(nodes ids lonely))
    local mac=$(macfunc $nfsserver)
    onadmin setup_nfs_server $mac
}

function prepareinstcrowbar
{
    echo "connecting to crowbar admin server at $adminip"
    ! [ -e crowbar.patch ] || $scp crowbar.patch root@$(wrap_ip $adminip):
    ! [ -e network.json.patch ] || $scp network.json.patch root@$(wrap_ip $adminip):
    onadmin prepareinstallcrowbar
    return $?
}

function bootstrapcrowbar
{
    echo "bootstrapping crowbar"
    onadmin bootstrapcrowbar
    return $?
}

function scp_install_chef_suse_override
{
    if [ -e "$install_chef_suse_override" ]; then
        ssh root@$adminip "mv /opt/dell/bin/install-chef-suse.sh{,.orig}"
        $scp -p "$install_chef_suse_override" \
            root@$(wrap_ip $adminip):/opt/dell/bin/install-chef-suse.sh
        ssh root@$adminip "chmod +x /opt/dell/bin/install-chef-suse.sh"
    fi
}

function instcrowbar
{
    scp_install_chef_suse_override
    onadmin installcrowbar
    local ret=$?
    $scp root@$(wrap_ip $adminip):$crowbar_install_log "$artifacts_dir/"
    return $ret
}

function instcrowbarfromgit
{
    scp_install_chef_suse_override
    safely rsync -av --exclude ".git" --exclude ".ci-tracking" ./crowbar root@$adminip:/root/
    onadmin installcrowbarfromgit
    local ret=$?
    $scp root@$(wrap_ip $adminip):$crowbar_install_log "$artifacts_dir/"
    return $ret
}

function mkvlan
{
    local defvlan=$1 ; shift
    local ip=$1 ; shift
    local intf=$cloudbr.$defvlan
    if (( $want_ipv6 > 0 )); then
        local netmask=$defaultnetmask
    else
        local netmask=24
    fi
    $sudo ip link add link $cloudbr name $intf type vlan id $defvlan
    safely $sudo ip link set $intf up
    if ! ip addr show $intf | grep -q $ip/$netmask; then
        safely $sudo ip addr add $ip/$netmask dev $intf
    fi
}

function setupvlangws
{
    # workaround https://bugzilla.novell.com/show_bug.cgi?id=845496
    echo 0 > /proc/sys/net/bridge/bridge-nf-call-iptables
    mkvlan $vlan_public $publicgw
    mkvlan $vlan_octavia $octaviagw
    if [[ $mkch_physcloudif ]] ; then
        $sudo brctl addif $cloudbr $mkch_physcloudif
        $sudo ip link set dev $mkch_physcloudif up
    fi
}

function shutdowncloud
{
    echo "Shutting down: $cloud"
    echo
    ${mkclouddriver}_do_shutdowncloud
}

function restartcloud
{
    vgchange -ay $cloudvg
    libvirt_net_start
    $sudo virsh start $cloud-admin
    setupvlangws
    wait_for_crowbar_ntpd
    local i
    for i in $(nodes ids all) ; do
        $sudo virsh start $cloud-node$i
    done
    wait_for_crowbar_ssh
}

# bring up VMs that will take cloud controller/compute/storage roles
function setupnodes
{
    onadmin wait_tftpd || return $?
    setupvlangws
    local i
    for i in $(nodes ids normal) ; do
        local macaddress=$(macfunc $i)

        # transport drdb volume information to admin node (needed for proposal of data cluster)
        drbd_serial=""
        if [ $drbd_hdd_size != 0 ]; then
            if [ $i -le 2 ] ; then
                drbd_serial="$cloud-node$i-drbd"
                # libvirt does not accept anything other than [:alnum:]_-
                # for serial strings:
                drbd_serial=${drbd_serial//[^A-Za-z0-9-_]/_}
                drbdnode_mac_vol="${drbdnode_mac_vol}+${macaddress}#${drbd_serial}"
                drbdnode_mac_vol="${drbdnode_mac_vol#+}"
            fi
        fi
        local mac_params=""
        for nicnumber in $(seq 1 $nics) ; do
            mac_params+=" --macaddress $(macfunc $i $nicnumber)";
        done
        local ironic_params=""
        [[ $want_ironic ]] && ironic_params="--ironicnic $ironicnic"
        local localrepos_params=""
        if [ -n "${localreposdir_src}" -a -n "${localreposdir_target}" ] ; then
            localrepos_params="--localreposrc ${localreposdir_src} --localrepotgt ${localreposdir_target}"
        fi

        local pcipassthrough_params=""
        if [[ $want_pci_passthrough = 1 ]]; then
            pcipassthrough_params="--pcipassthrough"
        fi

        local ipmi_params=""
        if [[ $want_ipmi = 1 ]] ; then
            ipmi_params="--ipmi"
            local bmc_addr=$(( 162 + $i ))
            if [[ $bmc_addr > 240 ]] ; then
                echo "You have more nodes than allocated addresses."
                exit 1
            fi
            bmc_addr=$net.$bmc_addr
            mkveth $i $bmc_addr
            generate_lan_config $i $bmc_user $bmc_password $bmc_addr
            screen -S $cloud-node$i-bmc-lan -d -m ipmi_sim -c /tmp/ipmi-lan-$cloud-node$i.conf -f ${SCRIPTS_DIR}/lib/ipmi/ipmi_sim.emu
        fi

        ${scripts_lib_dir}/libvirt/compute-config $cloud $i \
                        $mac_params \
                        $ironic_params \
                        $ipmi_params \
                        $pcipassthrough_params \
                        --cephvolumenumber $cephvolumenumber \
                        --drbdserial "$drbd_serial"\
                        --computenodememory $compute_node_memory \
                        --controllernodememory $controller_node_memory \
                        --libvirttype $libvirt_type \
                        --vcpus $vcpus \
                        --emulator $(get_emulator) \
                        --vdiskdir $vdisk_dir \
                        --bootorder 3 \
                        --numcontrollers $(get_nodenumbercontroller) \
                        $localrepos_params \
                        --firmwaretype "$firmware_type" \
                        --controller-raid-volumes $controller_raid_volumes > /tmp/$cloud-node$i.xml

        safely libvirt_vm_start /tmp/$cloud-node$i.xml
    done

    echo "========================================================"
    echo " Note: If you interrupt mkcloud now and want to proceed"
    echo "       later, make sure to run the following command:"
    echo
    echo " export drbdnode_mac_vol=\"${drbdnode_mac_vol}\""
    echo
    echo "========================================================"
    sleep 10

    # mkch_physwol can be set by the user to a space-separated list of MAC-addrs
    # of pysical nodes to wake up using WoL
    for i in $mkch_physwol ; do
        $sudo ether-wake -i $cloudbr $i
    done
    return 0
}

# allocate cloud nodes with an operating system
# and wait for nodes to reach the ready state
function instnodes
{
    safely onadmin allocate

    echo "Waiting for the installation of the nodes ..."
    safely onadmin waitcloud
    safely onadmin post_allocate
}

function deployexternalceph
{
    if [[ $want_external_ceph = 0 ]] ; then
        echo "\$want_external_ceph not set, skipping"
        return
    fi
    if [[ $(nodes number lonely) = 0 ]] ; then
        echo "\$nodenumberlonelynode = 0, can't build a ceph cluster of 0 nodes"
        return
    fi

    local new_nodes=()
    for i in $(nodes ids lonely) ; do
        local mac=$(macfunc $i)
        new_nodes=(${new_nodes[@]} $(mac_to_nodename $mac))
    done
    onadmin external_ceph ${new_nodes[@]}
}

function proposal
{
    onadmin proposal
    return $?
}

function install_vbmc
{
    # vbmc already available?
    vbmc --version 2> /dev/null && return 0

    # some simple checks before we try to add SP4-Cloud9 repo
    grep -q 12-SP[45] /etc/os-release || return 1

    # virtualbmc package is available in cloud9 repo
    $sudo zypper addrepo --priority 500 --refresh \
        http://download.suse.de/ibs/SUSE:/SLE-12-SP4:/Update:/Products:/Cloud9/standard/ \
        SLE-12-SP4-Cloud9
    $sudo zypper -n in python-virtualbmc
}

function start_vbmc
{
    local ironic_node=$1
    local ipmi_port=$2
    local ipmi_user=$3
    local ipmi_password=$4
    # make sure old vbmc is not interfering
    $sudo vbmc stop $ironic_node || true
    $sudo vbmc delete $ironic_node || true

    $sudo vbmc add $ironic_node --port $ipmi_port \
        --username $ipmi_user --password $ipmi_password
    $sudo vbmc start $ironic_node
}

function setup_ironic_test_env
{
    local ipmi_user=admin
    local ipmi_password=ipmipass

    local host_admin_ip=${net_admin}${ip_sep}1

    install_vbmc

    local i
    for i in $(nodes ids ironic) ; do
        local ironic_node=$cloud-node$i
        local ipmi_port

        # pick some free port from (arbitrary) 6230-7230 range
        # NOTE: if some other vbmc was already created for this port,
        # but it was not started yet, it might cause conflict later
        for ipmi_port in $(seq 6230 7230); do
            netstat -atun | grep -q ":$ipmi_port\s" || break;
        done

        wait_for 5 2 "start_vbmc $ironic_node $ipmi_port $ipmi_user $ipmi_password" 'vbmc to start successfully'

        # test if IPMI interface is working
        ipmitool -I lanplus -H $host_admin_ip -p $ipmi_port -U $ipmi_user -P $ipmi_password power status

        local ram_mb=$(( compute_node_memory / 1024 ))
        # create node in ironic
        onadmin create_ironic_node $cloud-node$i \
            $vcpus $ram_mb $ironicnode_hdd_size $(macfunc $i 1) \
            $host_admin_ip $ipmi_port $ipmi_user $ipmi_password
    done

    onadmin create_flavor b1.small $ram_mb $vcpus $(( $ironicnode_hdd_size - 1 ))
}

function teardown_ironic_test_env
{
    install_vbmc

    local i
    for i in $(nodes ids ironic) ; do
        local ironic_node=$cloud-node$i

        onadmin delete_ironic_node $cloud-node$i

        # cleanup vbmc for this ironic node
        $sudo vbmc stop $ironic_node || true
        $sudo vbmc delete $ironic_node || true
    done
}

function testsetup
{
    [[ $want_ironic = 1 ]] && setup_ironic_test_env

    onadmin testsetup
    local ret=$?

    [[ $want_ironic = 1 ]] && teardown_ironic_test_env

    $scp root@$(wrap_ip $adminip):tempest*.log "$artifacts_dir/"
    $scp root@$(wrap_ip $adminip):simple_horizon.log "$artifacts_dir/"

    # Register the cloud in Rally and import the results
    if [[ $rally_server && -f $artifacts_dir/tempest.subunit.log ]] ; then
        local title=$(testname)
        local type=$(testtype)

        $scp root@$(wrap_ip $adminip):.openrc "$artifacts_dir/"
        local tmpdir=`ssh $sshopts rally@$rally_server "mktemp -d rally-XXXXXX"`
        $scp "$artifacts_dir/tempest.subunit.log" "$artifacts_dir/.openrc" rally@$rally_server:$tmpdir
        ssh $sshopts rally@$rally_server "
            source rally/bin/activate
            source $tmpdir/.openrc
            deployment=\$(rally deployment create --fromenv --name=\"$title\" | awk '/Using deployment:/{print \$3}')
            rally verify import --set-name=\"$type\" --file=$tmpdir/tempest.subunit.log --deployment=\$deployment
            rm -fr $tmpdir
        "
    fi

    return $ret
}

function testpreupgrade
{
    onadmin testpreupgrade
}

function testpostupgrade
{
    onadmin testpostupgrade
}

function testname
{
    local -a var_names=(ha cluster upgrade ceph cinder drbd net
        netmode sles12 dvr rootfs)
    local -a var_values=("$hacloud" "$clusterconfig"
        "$upgrade_cloudsource" "$cephvolumenumber" "$cinder_backend"
        "$drbd_hdd_size" "$networkingplugin" "$networkingmode"
        "$want_sles12" "$want_dvr" "$want_rootfs")

    if [[ ${#var_names[@]} != ${#var_values[@]} ]] ; then
        complain 123 "testname error: arrays of different length, please check the code"
    fi

    local name="$cloudsource"
    local n i=0
    for n in ${var_names[@]} ; do
        if [[ ${var_values[$i]} ]] ; then
            name="$name/$n:${var_values[$i]}"
        fi
        i=$(($i+1))
    done

    # Check if this instance of mkcloud is running inside Jenkins
    if [[ $JENKINS_URL ]] ; then
        name="$BUILD_NUMBER - $name - $BUILD_URL"
    fi

    echo $name
}

function testtype
{
    # If `tempestoptions` contains '-s' or '--smoke' parameter, is a
    # smoke test.  If there is '--load-list 2015.03.required.txt' is a
    # defcore test.  A full test is assumed by default.
    local type="full"
    if [[ $tempestoptions ]] ; then
        if [[ $tempestoptions =~ -s ]] ; then
            type="smoke"
        elif [[ $tempestoptions =~ --load-list.*.required.txt ]] ; then
            type="defcore"
        elif [[ $tempestoptions =~ --load-list ]] ; then
            type="ad-hoc"
        elif [[ $tempestoptions =~ ironic ]] ; then
            type="ironic"
        fi
    fi

    echo $type
}

function rebootcrowbar
{
    # reboot the crowbar instance
    #  and test if everything is up and running afterwards
    sshrun "reboot"
    wait_for 50 3 "! nc -w 1 -z $adminip 22" 'crowbar to go down'
    wait_for_crowbar_ssh
    echo "waiting another 180 seconds for services"
    sleep 180
    sshrun "mount -a -t nfs" # workaround repos not mounted on reboot because NFS needs bind, but bind says it requires NFS
    return $?
}

function rebootcloud
{
    # reboot compute nodes
    #  and test if everthing is up and running afterwards
    onadmin rebootcloud
    return $?
}

function rebootcompute
{
    echo "WARNING: called deprecated rebootcompute step" >&2
    rebootcloud
    return $?
}

function rebootneutron
{
    onadmin rebootneutron
    return $?
}

function qa_test
{
    local ghsc=github.com/SUSE-Cloud
    mkdir -p ~/$ghsc/
    pushd ~/$ghsc/
    if [ -e "qa-openstack-cli" ] ; then
        cd qa-openstack-cli/
        git pull
    else
        git clone https://$ghsc/qa-openstack-cli.git
    fi
    popd
    rsync -av ~/$ghsc/qa-openstack-cli root@$(wrap_ip $adminip):
    onadmin qa_test
    ret=$?

    $scp -r root@$(wrap_ip $adminip):qa_test.logs/ $artifacts_dir/
    return $ret
}

function crowbarbackup
{
    safely onadmin crowbarbackup "$@"
    local btargetdir=${artifacts_dir:-.}
    mkdir -p "$btargetdir"
    safely $scp root@$(wrap_ip $adminip):/tmp/backup-crowbar.tar.gz "$btargetdir"
}

function crowbarrestore
{
    local btarball=backup-crowbar.tar.gz
    [ -e "$artifacts_dir/$btarball" ] && btarball="$artifacts_dir/$btarball"
    if [ ! -e "$btarball" ] ; then
        complain 56 "No crowbar backup tarball found."
    fi
    onhost_reset_admin
    safely $scp "$btarball" root@$(wrap_ip $adminip):/tmp/
    safely onadmin crowbarrestore "$@"
}

function onhost_reset_admin
{
    # deploy and setup a fresh admin node
    safely onhost_cleanup_admin_node
    safely onhost_prepareadmin
    safely setupadmin
    safely addupdaterepo
    safely prepareinstcrowbar
    safely bootstrapcrowbar
}

function crowbarupgrade_5plus
{
    if iscloudver 6plus ; then
        cloudsource=$upgrade_cloudsource onadmin prepare_cloudupgrade_nodes_repos
        iscloudver 6 && onadmin upgrade_ses_to_4
        onadmin zypper_patch_all
        onadmin upgrade_prechecks
        if [[ $cloudsource =~ devel ]] || [[ $upgrade_cloudsource =~ devel ]]; then
            onadmin allow_vendor_change_at_nodes
        fi
        export cloudsource=$upgrade_cloudsource
        onadmin prepare_crowbar_upgrade
        onadmin prepare_cloudupgrade_admin_repos
        onadmin addupdaterepo upgrade
        onadmin upgrade_admin_backup
        onadmin upgrade_admin_repocheck
        onadmin upgrade_admin_server
        wait_for 100 12 "! nc -w 10 -z $adminip 22" 'crowbar to go down after upgrade'
        wait_for_crowbar_ssh
        onadmin check_admin_server_upgraded
        # use crowbar-init to bootstrap crowbar (only for 6-7 upgrade)
        iscloudver 7 && onadmin bootstrapcrowbar "with_upgrade"
        onadmin wait_for_crowbar_api
    else
        onadmin prepare_crowbar_upgrade
        crowbarbackup "with_upgrade"
        export cloudsource=$upgrade_cloudsource
        crowbarrestore "with_upgrade"
    fi
}

function cloudupgrade
{
    if iscloudver 6plus; then
        local cloudver=$(getcloudver)
        # this must be set at the start of upgrade, before we switch to new cloudsource:
        export upgrade_progress_file="/var/lib/crowbar/upgrade/$cloudver-to-$((cloudver+1))-progress.yml"
        crowbarupgrade_5plus
        if [[ $want_ping_running_instances = 1 ]]; then
            onadmin ping_running_instances
        fi
        onadmin crowbar_nodeupgrade
    elif iscloudver 5plus ; then
        crowbarupgrade_5plus
        # upgrade and reapply all barclamps
        onadmin runlist crowbar_nodeupgrade \
                        proposal
    else
        onadmin runlist prepare_cloudupgrade \
                        cloudupgrade_1st \
                        cloudupgrade_2nd \
                        cloudupgrade_clients \
                        cloudupgrade_reboot_and_redeploy_clients
    fi
}

function prepareupgraderepos
{
    export cloudsource=$upgrade_cloudsource
    onadmin prepare_cloudupgrade_nodes_repos
    onadmin prepare_cloudupgrade_admin_repos
}

function cct
{
    onadmin run_cct "$@"
    return $?
}

function devsetup
{
    onadmin devsetup
    return $?
}

function setup_aliases
{
    onadmin setup_aliases
    return $?
}


function batch
{
    onadmin batch $scenario
    return $?
}

function show_steps
{
    cat <<EOSTEPS
Usage:
$0 <step> [<step>,...]

'step' is one of:
    $allcmds

These steps are expanding to the following steps:
    all
        -> $(expand_steps all)
    all_noreboot
        -> $(expand_steps all_noreboot)
    all_batch
        -> $(expand_steps all_batch)
    all_batch_noreboot
        -> $(expand_steps all_batch_noreboot)
    plain
        -> $(expand_steps plain)
    plain_with_upgrade
        -> $(expand_steps plain_with_upgrade)
    plain_with_upgrade_test
        -> $(expand_steps plain_with_upgrade_test)
    instonly
        -> $(expand_steps instonly)

Steps:
    cleanup:        kill all running VMs, zero out boot sectors of all LVM volumes
    prepare:        create LVM volumes, setup libvirt networks
    setupadmin:     create the admin node and install the cloud product
    prepareinstcrowbar: add repos and install crowbar packages
    bootstrapcrowbar: setup the postgres database and start crowbar
    instcrowbar:    install crowbar and chef on the admin node
    setupnodes:     create the nodes and let crowbar discover them
    instnodes:      allocate and install compute nodes
    onadmin:        run a step on the admin node - use as onadmin+func+param1
    proposal:       create and apply proposals for default setup
    setup_aliases:  set aliases for the nodes (usually needed for batch step)
    batch:          build proposals from exported crowbar_batch YAML file
    setuplonelynodes: boot a number (defined by nodenumberlonelynode) of non-crowbar registered nodes in the admin network
    setupironicnodes: boot a number of empty nodes to be used by the ironic service
    crowbar_register: register a number (defined by nodenumberlonelynode) of non-crowbar nodes with crowbar (setuplonelynodes needs to have run before)
    deployexternalceph: Deploy an external SES cluster on a number of crowbar nodes (defined by nodenumberlonelynode),
                    this step assumes that both setuplonelynodes and crowbar_register have been run first.
    testsetup:      start a VM in the cloud
    addupdaterepo:  addupdate repos defined in UPDATEREPOS= (URLs separated by '+')
    runupdate:      run zypper up on the crowbar node
                    (compute nodes are automatically updated via chef run)
    cloudupgrade:   upgrade the cloud to the version specified
                    by the upgrade_cloudsource parameter
    rebootcrowbar:  reboot the crowbar instance and wait for it being up
    rebootcloud:    reboot the cloud nodes and wait for them being up
    restartcloud:   start a pre-existing cloud again after host reboot
    createadminsnapshot: create snapshot of admin node disk
    restoreadminfromsnapshot: restore admin node disk from snapshot
    createcloudsnapshot:
                    create a snapshot of all nodes
    restorecloudfromsnapshot:
                    restore all nodes from snapshot
    devsetup:       installs the crowbar development environment in /opt/crowbar
    qa_test:        run the qa test suite
    help:           usage of steps and environment variables
    steps:          usage of steps only

EOSTEPS
}

function steps() {
    show_steps
    exit 1
}

function help
{
    show_steps
    cat <<EOUSAGE


Environment variables (need to be exported):

Mandatory
    cloudvg=vg0
        set the volume group name to create lvm volumes in
        Cloud volumes will be prefixed with the cloud name.
        The cleanup function will only cleanup volumes with this prefix.
    cloudpv=/dev/vdx (default /dev/vdb)
        Device where a LVM physical volume will be created, all data lost on this device
        Should be at least 80 GB. The volume group will be called 'cloud'.
    cloudsource=develcloud6/7/8/9 | GM6 | GM6+up | GM7 | GM7+up | GM8 | GM8+up | GM9 | GM9+up  (default '')
        NOTE: The latest version always is in development. So do NOT expect it to work out of the box.
        NOTE: If you need a stable/working version use <latest-version>-1.
        defines the installation source of the product
        develcloud6   : product from IBS Devel:Cloud:6
        develcloud7   : product from IBS Devel:Cloud:7
        develcloud8   : product from IBS Devel:Cloud:8
        develcloud9   : product from IBS Devel:Cloud:9
        GM6           : SUSE Cloud Goldmaster 6 without updates
        GM7           : SUSE Cloud Goldmaster 7 without updates
        GM8           : SUSE Cloud Goldmaster 8 without updates
        GM9           : SUSE Cloud Goldmaster 9 without updates
        GM6+up        : SUSE Cloud Goldmaster 6 with released maintenance updates
        GM7+up        : SUSE Cloud Goldmaster 7 with released maintenance updates
        GM8+up        : SUSE Cloud Goldmaster 8 with released maintenance updates
        GM9+up        : SUSE Cloud Goldmaster 9 with released maintenance updates
        M?*           : uses official Milestone? ISO image (? is a number)

Optional
    qa_crowbarsetup='path/to/script' (default: same directory as mkcloud is located in)
        set an optional path to qa_crowbarsetup.sh
    hacloud='' | 1  (default='')
        Set up a highly available cloud; requires to configure the clusters via clusterconfig='...'
    clusterconfig
        A string with a cluster configuration. The services for data, network and services cluster can
        be deployed in 1, 2 or 3 clusters. The configuration string looks like this:
        The string is: '<group1>:<group2>' (a ':' separated list of groups).
        A group is:    'clustername1+clustername2=clusternodecount'
        The first clustername of a group defines the name of the cluster.
        Where 'clusternodecount' is the number of nodes in the given cluster.
        Examples:
            3 clusters: clusterconfig='data=2:network=3:services=5'
            2 clusters: clusterconfig='services+data=2:network=3'
            1 cluster:  clusterconfig='data+network+services=2'
    upgrade_cloudsource='' (default='')
        set new cloudsource for upgrade process
    TESTHEAD='' | 1  (default='')
        use Media from Devel:Cloud:Staging (except for GM* targets)
    controller_raid_volumes (default=0)
        The number of disks to join into the software RAID for the controller node.
        Mimimal number to setup RAID is 2.
    cephvolumenumber  (default=1)
        the number of ceph volumes that will be created per node
        note: in SUSE OpenStack Cloud 5 do it manually
    cephvolume_hdd_size (default 21)
        Set the size in GB of data disk attached in compute nodes with enabled 'cephvolumenumber'
    controller_ceph_hdd_size (default 25)
        Set the size in GB of data disk attached in controller node  with enabled 'cephvolumenumber'
    lonelynode_hdd_size (default 20)
        Set the size in GB of data disk for lonely node.
    ironicnode_hdd_size (default 20)
        Set the size in GB of data disk for ironic node.
    nodenumber=2    (default 2)
        set the number of nodes to be created for the cloud (excluding admin node)
        In HA mode (hacloud=1) the nodes needed for clusters are subtracted from nodenumber; the
        remaining nodes are compute nodes.
    nodenumberlonelynode=2    (default 0)
        set the number of non-crowbar registered nodes to be created in the admin network
    nodenumberironicnode=2    (default 0)
        set the number of empty nodes to be used by the ironic service
    vcpus=1         (default $vcpus)
        set the number of CPU cores per compute node
    nics=1         (default $nics)
        set the number of network interfaces per cloud node
        if Ironic is enabled (want_ironic=1), default is set to 2
    adminvcpus=1    (default $adminvcpus)
        set the number of CPU cores for admin node
    admin_node_memory (default 4194304)
        Set the memory in KB assigned to admin node
    controller_node_memory (default 5242880)
        Set the memory in KB assigned to controller nodes
    compute_node_memory (default 2097152)
        Set the memory in KB assigned to compute nodes
    drbd_hdd_size  (default 0, or 15 if hacloud is set)
        Set the size in GB of the DRBD data disks attached to the
        nodes in the cluster hosting the database and rabbitmq.
    drbd_database_size (default 5)
        Set the size in GB of the DRBD LV to request the database barclamp
        to set up within the DRBD data disks attached to the nodes in the
        cluster hosting the database.
    drbd_rabbitmq_size (default 5)
        Set the size in GB of the DRBD LV to request the rabbitmq barclamp
        to set up within the DRBD data disks attached to the nodes in the
        cluster hosting RabbitMQ.
    networkingplugin
        Set the networking plugin to be used by neutron (e.g. openvswitch),
        if it isn't defined the barclamp-neutron's default is used.
    networkingmode
        Set the networking mode to be used by neutron (e.g. gre)
        if it isn't defined the barclamp-neutron's default is used.
    keep_existing_hostname=1    (default='')
        If this option is enabled crowbar_register keeps the existing
        hostname. When the crowbar_register option is enabled too then
        it generates and sets random hostnames.
    debug_mkcloud=1  (default 0)
        enable debug mode for mkcloud via 'set -x'
    debug_qa_crowbarsetup=1 (default 0)
        enable debug mode for qa_crowbarsetup.sh via 'set -x'
    debug_openstack=1 (default 0)
        enable debug mode for the openstack components
        sets debug true in the openstack proposals
    user_keyfile='path/to/file'
        path to optional user public ssh keyfile to inject into crowbar to
        /root/.ssh/authorized_keys file and to provisioner barclamp
    want_sles12=1 (default 0)
        setup SLE12 compute nodes
    want_dvr=1 (default='')
        if DVR support should be turned on by neutron barclamp. Only works with openvswitch and vxlan.
    want_rootfs=btrfs (default='')
        Deploy all discovered/pxe booted nodes with BTRFS as root filesystem. if empty,
        an intrinsic default is chosen. Only possible with SLE12 node.
    want_all_debug=1 (default='')
        Enable debug level logging in barclamp proposals for all proposals.
    want_\$PROPOSAL_proposal=[01]
        All proposals are enabled by default, except those listed further below
        Set to 0 to prevent \$PROPOSAL from being deployed. e.g.
        want_aodh_proposal=0
        want_barbican_proposal=0
        want_sahara_proposal=0
        The following proposals are default off:
        Set to 1 to enable \$PROPOSAL to be deployed. e.g.
        want_designate_proposal=1 (Cloud9+ only)
        want_magnum_proposal=1
        want_monasca_proposal=1   (Cloud7+ only)
        want_octavia_proposal=1 (Cloud9+ only)
        want_trove_proposal=1
    want_horizon_integration_test=1 (default='')
        Execute selenium integration tests for Horizon after Tempest. This test is in beta, and
        a fail will not be considered for now.
    want_efi=1 (default=0)
        Deploy nodes with EFI. Only available from Cloud7 and later.
    want_keystone_token_type='uuid' | 'fernet' (default='')
        Choose Keystone token provider type, possible values are "uuid" or "fernet".
        Default value will be applied from Keystone barclamp proposal.
    want_cd='' | 1  (default='')
        Set up a cloud based on the continuous delivery channel.
    want_ceph=1
        Setup an internal ceph cluster using the ceph barclamp.
    want_external_ceph=1 (default=0)
        Setup an external ceph cluster on a set of recently registered lonely nodes
    want_ses_version=5 (default=0)
        Specify what version of SES you want to use. The value is the SES version (4 = SES4, 5 = SES5)
    want_ironic=1 (default='')
        Prepare (virtualized) environment for ironic.
    ironicnic=1 (default=1)
        Index of network card to be used for ironic.
    install_chef_suse_override='path/to/script'
        Optional path to an alternate version of install-chef-suse.sh on the mkcloud hostto use
        instead of the one from the packages.  This will be scp'd to the admin node before use.
    vlan_public=<id> (default 300)
        VLAN id for public network
    tempestoptions (default='-t -s' for <cloud7 and '--smoke' for >=cloud7)
        parameters passed to run_tempest.sh script
    rally_server (default='')
        rally server address
    cct_tests='test1+test2' (default='features')
        Defines which cct_tests should be run while the cct step.
    scenario='scenario.yaml' (default='')
        Defines which scenario file should be used.
        Currently only the step 'batch' uses such a file.
    scenario_dir='/tmp' (default=${SCRIPTS_DIR}/scenarios/cloud\$(getcloudver)/)
        Full path to directory containing scenario file.
    log_dir=PATH (default = ${log_dir})
        The standard output and error will be logged in that directory.
    cache_clouddata (default='')
        If set, mkcloud will maintain on the host a local cache of clouddata related
        artifacts (repositories images) and avoid downloading or require a
        permanent network connection after the initial seed is complete. Setting
        this is a convenience for all the individual options like localreposdir_*.
    localreposdir_src (default='')
        If you want to use repositories from your host's local filesystem, set
        this variable to point to the root of the hierarchy.
    want_all_ssl (default='0')
        If set, enables TLS for all services. Can also enable TLS for
        individual services using want_{proposal}_ssl.
    want_ssl_trusted (default='1' for cloud7+)
        If set, does not use the --insecure flag in openstack CLI commands.
    want_monasca_tsdb (Cloud9+ only)
        Allows setting time-series DB used for Monasca [influxdb|cassandra].
    want_ipv6 (Currently in development)
        Prepare crowbar to use a full IPv6 single stack control plane. Enabling will also set
        the firmware_type to UEFI as it's required to netboot nodes over IPv6.
EOUSAGE
    qacrowbarsetup_help
    exit 1

# UNDOCUMENTED OPTIONS:
#
# virtualcloud
# cloudfqdn
# forwardmode
# net_public
# net_admin
# adminnetmask
# adminip
# admingw
# cpuflags
# admin_node_memory
# controller_node_memory
# compute_node_memory
# hyperv_node_memory
# adminnode_hdd_size
# controller_hdd_size
# computenode_hdd_size
# cloudbr
# libvirt_type
# firmware_type
# localreposdir_target
# wipe

}


function addupdaterepo
{
    onadmin addupdaterepo "$@"
    return $?
}

function runupdate
{
    onadmin runupdate
    return $?
}

function is_concurrent_run
{
    [ -e $pidfile ] && kill -0 `cat $pidfile` 2>/dev/null && return 0
    echo $$ > $pidfile
    return 1
}

function sanity_checks
{
    if [[ `$sudo id -u` != 0 ]] ; then
        complain 1 "This script needs to be run as root or have sudoers setup" \
            "Please be aware that this script will create a LVM" \
            "and kill all current VMs on this host."
    fi

    if is_concurrent_run; then
        complain 33 "mkcloud was started twice from same working directory: `pwd`" \
            "Please always use a separate working directory for each (parallel) mkcloud run."
    fi

    if [[ $want_sap ]] ; then
        complain 34 "Usage of obsolete parameter. want_sap is now want_cd (continuous delivery)."
    fi

    # test for existence of qa_crowbarsetup.sh
    if [ ! -e $SCRIPTS_DIR/qa_crowbarsetup.sh ] ; then
        complain 87 "qa_crowbarsetup.sh not found.
            Make sure to call mkcloud from your git clone."
    fi

    if [ -z "$cloudsource" ] ; then
        echo "Please set the env variable:"
        echo "export cloudsource=M?*|develcloud6|develcloud7|develcloud8|develcloud9|GM6|GM6+up|GM7|GM7+up|GM8|GM9|GM9+up"
        exit 1
    fi

    # checking clusterconfig
    if [[ $hacloud = 1 && ! $clusterconfig ]] ; then
        echo "Examples for clusterconfig:"
        echo '3 clusters: clusterconfig="data=2:network=3:services=3"'
        echo '2 clusters: clusterconfig="services+data=2:network=3"'
        echo '1 cluster:  clusterconfig="data+network+services=2"'
        complain 70 "No cluster config provided for HA setup."
    fi

    ${mkclouddriver}_do_sanity_checks

    if ! /usr/sbin/iptables --help | grep -q -- --wait ; then
        complain 90 "mkcloud requires a more recent iptables version that supports --wait"
    fi
    if [ -e /etc/init.d/SuSEfirewall2_init ] && rcSuSEfirewall2 status ; then
        complain 91 "SuSEfirewall is running - it will interfere with the iptables rules done by libvirt" \
            "Please stop the SuSEfirewall completely and run mkcloud again" \
            "Run:  rcSuSEfirewall2 stop && insserv -r SuSEfirewall2_setup && insserv -r SuSEfirewall2_init"
    fi

    if grep "devpts.*[^x]mode=.00" /proc/mounts ; then
        complain 13 "/dev/pts is not accessible for libvirt, maybe you use autobuild on your system." \
            "Please remount it using the following command:" \
            " # mount -o remount,mode=620,gid=5 devpts -t devpts /dev/pts"
    fi

    if [[ $wantedcmds =~ "setuplonelynodes" || $wantedcmds =~ "crowbar_register" ]] ; then
        if [[ $(nodes number lonely) -lt 1 ]] ; then
            complain 80 "Please set nodenumberlonelynode."
        fi
    fi

    case $cinder_backend in
        ''|netapp|local|nfs|raw|rbd)
            ;;
        *)
            complain 101 "$cinder_backend as the cinder backend is currently not supported."
            ;;
    esac

    if [[ $want_sles12 = 1 ]] && [[ $want_ceph = 1 ]] && [[ $(nodes number normal) -lt 3 ]] ; then
        complain 113 "Please increase number of nodes for this setup, minimal nodenumber=3"
    fi

    [[ $hacloud = 1 ]] && [[ $(uname -m) = aarch64 ]] && complain 7 "there are no HA repos for aarch64"
    if iscloudver 6plus && [[ $want_ceph = 1 ]] && [[ $(nodes number normal) -lt 3 ]] ; then
        complain 113 "Please increase number of nodes for this setup, minimal nodenumber=3"
    fi

    if [[ $want_ceph = 1 ]] && [[ $want_external_ceph = 1 ]] ; then
        complain 118 "\$want_ceph and \$want_external_ceph are mutully exclusive. Please choose one or the other"
    fi

    if [[ " ${steplist[*]} " == *" batch "* ]] ; then
        : ${scenario_dir:="${SCRIPTS_DIR}/scenarios/cloud$(getcloudver)"}
        need_scenario=1
        if [[ ! $scenario ]] ; then
            complain 114 "scenario parameter for batch step must point to a file in $scenario_dir"
        fi
        scenario_path="$scenario_dir/$scenario"
        if [[ ! -f $scenario_path ]]; then
            complain 115 "Scenario file not found at $scenario_path"
        fi
    fi
}


## MAIN ##
step_aliases="_new_admin _compute _upgrade _testupdate"

allcmds="$step_aliases _test_setuphost \
    all all_noreboot all_batch all_batch_noreboot instonly \
    plain plain_with_upgrade cleanup setuphost prepare libvirt_prepare setupadmin \
    prepareinstcrowbar bootstrapcrowbar instcrowbar instcrowbarfromgit setupnodes \
    setupcompute instnodes instcompute proposal testsetup rebootcrowbar \
    rebootcloud addupdaterepo runupdate testupdate \
    crowbarbackup crowbarrestore shutdowncloud restartcloud qa_test help \
    rebootneutron cloudupgrade prepareupgraderepos \
    setuplonelynodes crowbar_register createadminsnapshot \
    lonelynode_nfs_server setupironicnodes\
    restoreadminfromsnapshot createcloudsnapshot restorecloudfromsnapshot \
    cct steps batch setup_aliases onadmin onhost devsetup plain_with_upgrade_test \
    testpreupgrade testpostupgrade deployexternalceph"
wantedcmds=$@

function expand_steps
{
    # parse the commands and expand the aliases
    local runcmds=''
    local localwantedcmds=$@
    local cmd
    for cmd in $localwantedcmds ; do

        local found=0
        local onecmd
        for onecmd in $allcmds ; do
            if [[ $cmd =~ ^$onecmd(\+.+)?$ ]] ; then
                found=1
                case "$cmd" in
                    setupcompute|instcompute)
                        corrected=${cmd/compute/nodes}
                        echo "WARNING: the '$cmd' step is deprecated; use '$corrected' instead." >&2
                        cmd=$corrected
                        ;;
                esac

                case "$cmd" in
                    _new_admin)
                        runcmds="$runcmds cleanup prepare setupadmin addupdaterepo prepareinstcrowbar runupdate bootstrapcrowbar instcrowbar"
                    ;;
                    _compute)
                        runcmds="$runcmds setupnodes instnodes setup_aliases proposal"
                    ;;
                    all)
                        runcmds="$runcmds `expand_steps _new_admin` rebootcrowbar `expand_steps _compute` testsetup rebootcloud"
                    ;;
                    all_noreboot)
                        runcmds="$runcmds `expand_steps _new_admin` `expand_steps _compute` testsetup"
                    ;;
                    all_batch)
                        runcmds="$runcmds `expand_steps _new_admin` rebootcrowbar setupnodes instnodes setup_aliases batch testsetup rebootcloud"
                    ;;
                    all_batch_noreboot)
                        runcmds="$runcmds `expand_steps _new_admin` setupnodes instnodes setup_aliases batch testsetup"
                    ;;
                    _testupdate|testupdate)
                        runcmds="$runcmds addupdaterepo runupdate testsetup"
                    ;;
                    _test_setuphost)
                        runcmds="$runcmds cleanup prepare setupadmin onadmin+test_setuphost"
                    ;;
                    plain)
                        runcmds="$runcmds `expand_steps instonly` proposal"
                    ;;
                    instonly)
                        runcmds="$runcmds cleanup prepare setupadmin prepareinstcrowbar bootstrapcrowbar instcrowbar setupnodes instnodes setup_aliases"
                    ;;
                    _upgrade)
                        runcmds="$runcmds cloudupgrade"
                    ;;
                    plain_with_upgrade)
                        runcmds="$runcmds `expand_steps plain` addupdaterepo runupdate `expand_steps _upgrade`"
                    ;;
                    plain_with_upgrade_test)
                        runcmds="$runcmds `expand_steps plain` testpreupgrade addupdaterepo runupdate `expand_steps _upgrade` testpostupgrade"
                    ;;
                    *)
                        runcmds="$runcmds $cmd"
                    ;;
                esac
            fi
        done
        [ $found == 0 ] && complain - "Step $cmd not found." && return

    done
    runcmds=${runcmds## }
    runcmds=${runcmds%% }
    echo "${runcmds//  / }"
}

steplist=`expand_steps $wantedcmds`

: ${steplist:=help}
if [[ $steplist =~ ^(cleanup|help|steps|setuphost|shutdowncloud)$ ]] ; then
    # skipping sanity checks
    $steplist
    exit $?
fi
sanity_checks

echo "You choose to run these mkcloud steps:"
echo "  $steplist"
echo
sleep 2

for cmd in `echo $steplist` ; do
    echo
    echo "============> MKCLOUD STEP START <============"
    # next line is for jenkins log parser:
    echo "MKCLOUD step: $cmd"
    step_start=$(date +%s)
    echo "step started at `date '+%Y-%m-%d %H:%M:%S' -d@$step_start`: $cmd"
    echo
    sleep 2
    echo $cmd >> mkcloud.steps.log
    # support calls to functions with parameters
    # this is currently used for the cct function
    IFS=+
    cmd_and_parameters=($cmd) # split at + and convert to list
    unset IFS
    cmd=${cmd_and_parameters[0]}
    TIMEFORMAT="timing for mkcloud step '$cmd' real=%R user=%U system=%S"
    time "${cmd_and_parameters[@]}"
    ret=$?
    if [ $ret != 0 ] ; then
        set +x
        echo
        echo '$h1!!'
        echo "Error detected. Stopping mkcloud."
        echo "The step '$cmd' returned with exit code $ret"
        echo "Please refer to the $cmd function in this script when debugging the issue."
        error_exit $ret ""
    fi >&2
    echo
    step_end=$(date +%s)
    echo "step finished at `date '+%Y-%m-%d %H:%M:%S' -d@$step_end`: $cmd"
    log_timing "$step_start" "$step_end" "step" "$cmd"
    echo "^^^^^^^^^^^^= MKCLOUD STEP DONE: $cmd =^^^^^^^^^^^^"
    echo
done

pre_exit_cleanup
show_environment