mtps-pkg-test

#!/usr/bin/bash -efu

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See LICENSE for more details.
#
# Copyright: Red Hat Inc. 2018, 2023
# Author: Andrei Stepanov <astepano@redhat.com>


# We are testing RHEL compose and a Brew build. Compose is a set of installed
# packages.
# Some tests can fail.
#
#   * remove - can fail, for example for systemd.
#   * install - to install a package it is necessary to remove old version.
#
# Hard tests:
#
#   * update
#   * downgrade
#
# There is always old version of package at:
#
#   * https://download.devel.redhat.com/nightly/latest-RHEL-8/compose/BaseOS
#   * https://download.devel.redhat.com/nightly/latest-RHEL-8/compose/AppStream
#

: "${PROG:=${0##*/}}"
: "${ALLOWERASING:=--allowerasing}"

EXIT_CODE_SKIP=7
EXIT_CODE_WARN=8

# Source `mtps-setup' from $PATH
if command -v "mtps-setup" >/dev/null; then source "mtps-setup"; fi
# If previous fails source `mtps-setup` from this script dir
if [ -z "${YUMDNFCMD:-}" ]; then source "$(dirname "${BASH_SOURCE[0]}")/mtps-setup" || ( echo "Cannot source mtps-setup" >&2; exit 91 ); fi

YUM_VERSION=$("${YUMDNFCMD}" --version)
[ ${YUM_VERSION:0:1} == "3" ] && ALLOWERASING=""

from_nevra() {
    # Input: foo-1.0-1.i386 or foo-1:1.0-1.i386
    local nevra="$1" && shift
    # One of: name, ver, rel, epoch, arch
    local req_info="$1" && shift
    # Arch
    local arch="${nevra##*.}"
    debug "$nevra arch: $arch"
    [[ $req_info == 'arch' ]] && echo "$arch" && return
    # Release
    local rel="${nevra##*-}"
    rel="${rel%.*}"
    debug "$nevra rel: $rel"
    [[ $req_info == 'rel' ]] && echo "$rel" && return
    # Version
    local epoch_ver="${nevra%-${rel}.${arch}}"
    epoch_ver="${epoch_ver##*-}"
    local ver="${epoch_ver##*:}"
    debug "$nevra ver: $ver"
    [[ $req_info == 'ver' ]] && echo "$ver" && return
    # Epoch
    local epoch="${epoch_ver%${ver}}"
    epoch="${epoch//:/}"
    epoch="${epoch:-0}"
    debug "$nevra epoch: ${epoch}"
    [[ $req_info == 'epoch' ]] && echo "$epoch" && return
    # Name
    local name="${nevra%-${epoch_ver}-${rel}.${arch}}"
    debug "$nevra name: $name"
    [[ $req_info == 'name' ]] && echo "$name" && return
}

# Is the first arg (nevra) older than the second arg (nevra)?
nevra_older_then() {
    # Verify with: rpmdev-vercmp
    local nevra1="$1" && shift
    local nevra2="$1" && shift
    local name1="$(from_nevra "$nevra1" "name")"
    local name2="$(from_nevra "$nevra2" "name")"
    local epoch1="$(from_nevra "$nevra1" "epoch")"
    local epoch2="$(from_nevra "$nevra2" "epoch")"
    local version1="$(from_nevra "$nevra1" "ver")"
    local version2="$(from_nevra "$nevra2" "ver")"
    local release1="$(from_nevra "$nevra1" "rel")"
    local release2="$(from_nevra "$nevra2" "rel")"
    local arch1="$(from_nevra "$nevra1" "arch")"
    local arch2="$(from_nevra "$nevra2" "arch")"
    nevra1="${name1}-${epoch1}:${version1}-${release1}"
    nevra2="${name2}-${epoch2}:${version2}-${release2}"
    local rcmp
    # https://rpm-software-management.github.io/rpm/manual/lua
    # rpm.vercmp(v1, v2) returns -1, 0 or 1 if v1 is smaller, equal or larger than v2
    rcmp="$(rpm --define "pkg1 $nevra1" --define "pkg2 $nevra2"  --eval '%{lua: print(rpm.vercmp(rpm.expand("%pkg1"), rpm.expand("%pkg2")))}')"
    if [[ "$rcmp" == "-1" ]]; then
        # True
        return 0
    fi
    # False
    return 1
}

run_with_scriptlet_check() {
    # Run a command and check if the output doesn't contain certain keywords.
    # The return code from the command is preserved. If the command failed,
    # we don't bother checking the output. If the command succeeded, we check
    # the output and return "1" if any of the keywords were found in the output.
    #
    # Example usage:
    # run_with_scriptlet_check dnf -y install mypackage

    local cmd_output
    cmd_output=$(mktemp)
    debug "${FUNCNAME[0]}: running command \"$*\" and checking the output for errors."
    # Capture the output in a temporary file
    "$@" 2>&1 | tee "$cmd_output"
    # Preserve return code from the command
    local ret=${PIPESTATUS[0]}
    cmd_output_parts_prefix="cmd-output-part"
    if [ "$ret" -eq 0 ]; then
        # The script is running with globbing off, but it's useful here, so let's enable it temporarily
        set +f
        rm -f "$cmd_output_parts_prefix"*
        # Split the output into separate files on the lines containing the "Running scriptlet: " string
        csplit "$cmd_output" --quiet --prefix="$cmd_output_parts_prefix" --suffix-format='%04d' '/Running scriptlet: /' '{*}'

        # Check all output parts and find those that belong to the NVRA that is currently being tested
        for output_part in "$cmd_output_parts_prefix"*; do
            # Extract NVRA. The first line of the file should look like this:
            #   Running scriptlet: abc-1-1.el9.x86_64  50/180
            scriptlet_nvra=$(head -1 "$output_part" | awk -F' ' '{ print $3 }')

            # Remove epoch from the NEVRA under test
            test_nvra=$(echo "$NEVRA" | sed -e 's/-[0-9]\+:/-/')
            # Check if the  NVRA matches the NVRA under test
            if [[ "$test_nvra" == "$scriptlet_nvra" ]]; then
                # We found a part of the output that matters, let's look for problematic words here;
                # However, let's filter out lines with known false positives first:
                #  Ignore warning about a new .rpmsave file (https://access.redhat.com/solutions/60263)
                grep -P -v -i 'warning: .* saved as .*\.rpmsave' "$output_part" > "${output_part}-filtered" && mv "${output_part}-filtered" "$output_part"
                #  Ignore warning about UID being outside of the SYS_UID_MIN/SYS_UID_MAX range.
                #  This is a "known unfixable". See comments in OSCI-4825 for more information.
                grep -P -v -i 'useradd warning: [^ ]+ uid [0-9]+ outside of the SYS_UID_MIN [0-9]+ and SYS_UID_MAX [0-9]+ range\.' "$output_part" > "${output_part}-filtered" && mv "${output_part}-filtered" "$output_part"
                #  Ignore (bogus?) warnings about removed systemd unit files being changed on disk and needing to be reloaded.
                #  This seems to be a bug in systemd(?): https://github.com/systemd/systemd/issues/32959
                #  TODO: Remove this once systemd gets updated and the problem is no longer present
                grep -P -v -i "warning: the unit file, source configuration file or drop-ins of .* changed on disk\. run 'systemctl daemon-reload' to reload units\." "$output_part" > "${output_part}-filtered" && mv "${output_part}-filtered" "$output_part"

                # We only look for the following problematic words (case-insensitive):
                # * error
                # * failure
                # * failed
                # * warning
                # Note: we ignore cases where there is a leading or trailing dash (-),
                # the reason is that the keyword ("error" for example) can appear in package names.
                # See how the regexp works (and play with it!): https://regex101.com/r/Yws0tT/1
                grep -P -i -q '(?<![-a-z])error|failure|failed|warning(?![-a-z])' "$output_part" && ret=$EXIT_CODE_WARN
                if [ "$ret" -eq $EXIT_CODE_WARN ]; then
                    echo
                    echo
                    echo "***"
                    echo "The test succeeded, but potential scriptlet problems were detected in the output."
                    echo "Please check the log above for more information."
                    echo "***"
                    break
                fi
            fi
        done
    fi
    # Clean up the temporary file(s)
    rm -f "$cmd_output"
    rm -f "$cmd_output_parts_prefix"*

    # Globbing off again
    set -f

    return "$ret"
}

pkg_can_be_removed() {
    local nevra="$1" && shift
    local name="$(from_nevra "$nevra" "name")"
    debug "${FUNCNAME[0]}: checking whether $name can be removed with help of ${YUMDNFCMD}."
    "$YUMDNFCMD" --assumeyes --setopt "tsflags=test" "remove" "$name" >"$DEBUG_OUT" 2>&1
    if [ ${PIPESTATUS[0]} -eq 0 ]; then
        debug "${FUNCNAME[0]}: $name can be removed."
        return 0
    fi
    debug "${FUNCNAME[0]}: $name cannot be removed."
    return 1
}

pkg_remove_any() {
    local nevra="$1" && shift
    local name="$(from_nevra "$nevra" "name")"
    debug "${FUNCNAME[0]}: $name"
    if pkg_is_absent "$name"; then
        debug "${FUNCNAME[0]}: $name is not installed. Not removing."
        return 0
    fi
    # yum remove kernel-rt-kvm
    #=============================================================================
    # Package         Arch    Version                Repository              Size
    #=============================================================================
    #Removing:
    # kernel-rt-kvm   x86_64  4.18.0-39.rt9.82.el8   @rhel-8-buildroot      361 k
    # kernel-rt-kvm   x86_64  4.18.0-37.rt9.80.el8   @rhel-8-buildroot-old  361 k
    "$YUMDNFCMD" -y remove "$name" || :
    # yum returns 1 for removing non-existing package for yum4/dnf
    local ret=0
    pkg_is_present "$name" && ret=1
    return $ret
}

pkg_install_if_absent() {
    local nevra="$1" && shift
    local name="$(from_nevra "$nevra" "name")"
    debug "${FUNCNAME[0]}: $name"
    if pkg_is_present "$name"; then
        debug "${FUNCNAME[0]}: package $name is present. Not installing $nevra."
        return 0
    fi
    if ! "$YUMDNFCMD" -y install ${ALLOWERASING} "$nevra"; then
        echo "Installation of $nevra failed."
        echo "See $YUMDNFCMD command output above for more information."
        return 1
    fi
    if pkg_is_absent "$name"; then
        echo "Package $name is absent after installation."
        return 1
    fi
    return 0
}

pkg_is_present() {
    local name="$1" && shift
    debug "${FUNCNAME[0]}: $name"
    local ret=
    rpm -q --qf "" "$name" >/dev/null 2>&1 && ret=0 || ret=1
    debug "Package $name is $(test -z "${ret##0}" && echo present || echo absent)"
    return $ret
}

pkg_is_absent() {
    local name="$1" && shift
    local ret=
    pkg_is_present "$name" && ret=1 || ret=0
    return $ret
}

rpm_db_verify() {
    # Run "rpm --verify" on the installed package
    local name="$1" && shift
    debug "${FUNCNAME[0]}: $name"
    local ret=

    echo
    echo
    echo "***"
    echo "Verifying that the installed files match the metadata in the RPM database..."
    echo "$ rpm --verify $name"
    rpm --verify "$name" && ret=0 || ret=1
    echo
    if [ "$ret" != "0" ]; then
        echo "Some discrepancies were found. This is how to interpret the output above:"
        echo
        echo "S file Size differs"
        echo "M Mode differs (includes permissions and file type)"
        echo "5 digest (formerly MD5 sum) differs"
        echo "D Device major/minor number mismatch"
        echo "L readLink(2) path mismatch"
        echo "U User ownership differs"
        echo "G Group ownership differs"
        echo "T mTime differs"
        echo "P caPabilities differ"
        echo
        echo "These findings might be real bugs that can be fixed in the spec file,"
        echo "or they might be false positives that can be suppressed. See the \"%verify\""
        echo "scriptlet description in the RPM docs: https://rpm-software-management.github.io/rpm/manual/spec.html"
    else
        echo "(ok)"
    fi
    echo "***"

    return $ret
}

get_installed_nevra_newest() {
    # More packages with the same name can be installed
    # rpm -q "kernel-rt-kvm"
    # kernel-rt-kvm-4.18.0-37.rt9.80.el8.x86_64
    # kernel-rt-kvm-4.18.0-39.rt9.82.el8.x86_64
    # Return the newest one.
    local name="$1" && shift
    local nevras=($(rpm -q --qf "%{name}-%{epochnum}:%{version}-%{release}.%{arch}\n" "$name"))
    nevra="${nevras[0]}"
    for n in "${nevras[@]}"; do
        if nevra_older_then "$nevra" "$n"; then
            debug "${FUNCNAME[0]}: $nevra < $n"
            nevra="$n"
        else
            debug "${FUNCNAME[0]}: $nevra > $n. Skip"
        fi
    done
    debug "Newest package: $nevra"
    echo "$nevra"
}

get_installed_nevra_oldest() {
    # More packages with the same name can be installed
    # rpm -q "kernel-rt-kvm"
    # kernel-rt-kvm-4.18.0-37.rt9.80.el8.x86_64
    # kernel-rt-kvm-4.18.0-39.rt9.82.el8.x86_64
    # Return the oldest one.
    local name="$1" && shift
    local nevras=($(rpm -q --qf "%{name}-%{epochnum}:%{version}-%{release}.%{arch}\n" "$name"))
    nevra="${nevras[0]}"
    for n in "${nevras[@]}"; do
        if ! nevra_older_then "$nevra" "$n"; then
            debug "${FUNCNAME[0]}: $nevra > $n"
            nevra="$n"
        else
            debug "${FUNCNAME[0]}: $nevra < $n. Skip"
        fi
    done
    debug "Oldest package: $nevra"
    echo "$nevra"
}

pkg_downgrade_if_present() {
    local nevra="$1" && shift
    local name="$(from_nevra "$nevra" "name")"
    if pkg_is_absent "$name"; then
        echo "Package $nevra is absent. Not downgrading."
        return 0
    fi
    # installonlypkg(foo) case
    local installed_nevra_old="$(get_installed_nevra_oldest "$name")"
    echo "The (oldest) installed package: $installed_nevra_old"
    if nevra_older_then "$installed_nevra_old" "$nevra"; then
        echo "Not downgrading package: $nevra. There is already installed old package: $installed_nevra_old"
        return 0
    fi
    "$YUMDNFCMD" -y downgrade "$ALLOWERASING" "$name"
}

get_old_nevra() {
    local nevra="$1" && shift
    local name="$(from_nevra "$nevra" "name")"
    local version="$(from_nevra "$nevra" "ver")"
    local epoch="$(from_nevra "$nevra" "epoch")"
    local release="$(from_nevra "$nevra" "rel")"
    local arch="$(from_nevra "$nevra" "arch")"
    # Find previous version of the package.
    local old_nevras=($(
        repoquery -q "$name"."$arch" --qf="%{name}-%{epoch}:%{version}-%{release}.%{arch}" --show-duplicates | \
        tr -s ' ' | \
        grep "$arch" | \
        sed -n -e "/^${name}-${epoch}:${version}-${release}.${arch}$/"'!p'
        ))
    if [ ${#old_nevras[@]} -eq 0 ]; then
        debug "Cannot find older version for $nevra."
        return 1
    fi
    local old_nevra="${old_nevras[0]}"
    for n in "${old_nevras[@]}"; do
        if ! nevra_older_then "$old_nevra" "$n"; then
            debug "${FUNCNAME[0]}: $old_nevra > $n"
            old_nevra="$n"
        else
            debug "${FUNCNAME[0]}: $old_nevra < $n. Skip"
        fi
    done
    if ! nevra_older_then "$old_nevra" "$nevra"; then
        debug "Cannot find older version for $nevra."
        return 1
    fi
    debug "${FUNCNAME[0]}: older for $nevra is: $old_nevra"
    echo "$old_nevra"
}

pkg_remove_any_newer() {
    local nevra="$1" && shift
    local name="$(from_nevra "$nevra" "name")"
    local installed_nevra=($(rpm -q --qf "%{name}-%{epochnum}:%{version}-%{release}.%{arch}\n" "$name"))
    for to_remove in "${installed_nevra[@]}"; do
        if nevra_older_then "$to_remove" "$nevra"; then
            debug "Not removing older package: $to_remove"
            continue
        fi
        echo "Removing package: $to_remove"
        "$YUMDNFCMD" -y remove "$to_remove" || :
    done
}

pkg_remove_any_older() {
    local nevra="$1" && shift
    local name="$(from_nevra "$nevra" "name")"
    local installed_nevra=($(rpm -q --qf "%{name}-%{epochnum}:%{version}-%{release}.%{arch}\n" "$name"))
    for to_remove in "${installed_nevra[@]}"; do
        if ! pkg_can_be_removed "$to_remove"; then
            # RHBZ: 1698145
            echo "Package $to_remove cannot be removed. Skipping."
            continue
        fi
        if nevra_older_then "$to_remove" "$nevra"; then
            echo "Removing older package: $to_remove"
            "$YUMDNFCMD" -y remove "$to_remove" || :
        else
            debug "Not removing newer package: $to_remove"
            continue
        fi
    done
}

pkg_update_to_new_if_present() {
    local nevra="$1" && shift
    local name="$(from_nevra "$nevra" "name")"
    debug "${FUNCNAME[0]}: $name"
    if pkg_is_absent "$name"; then
        debug "Package $name is absent. Not updating."
        return 0
    fi
    # Skip update if installed package is not older than requested
    local installed_nevra_new="$(get_installed_nevra_newest "$name")"
    debug "(Newest) installed package: $installed_nevra_new"
    if ! nevra_older_then "$installed_nevra_new" "$nevra"; then
        debug "Installed package $installed_nevra_new is not older. Not updating."
        return 0
    fi
    "$YUMDNFCMD" -y upgrade "$ALLOWERASING" "$nevra"
    if pkg_is_absent "$nevra"; then
        echo "Could not update to: $nevra"
        return 1
    fi
    return 0
}

pkg_downgrade_to_old() {
    local nevra="$1" && shift
    local name="$(from_nevra "$nevra" "name")"
    if pkg_is_absent "$name"; then
        debug "Package $name is absent. Not downgrading."
        return 1
    fi
    local installed_nevra_old="$(get_installed_nevra_oldest "$name")"
    if rpm -q --provides "$installed_nevra_old" | grep -q -s "installonlypkg"; then
        echo "Installed $installed_nevra_old has RPM flag: installonlypkg. Not downgrading."
        return 0
    fi
    if nevra_older_then "$installed_nevra_old" "$nevra"; then
        echo "Not downgrading package: ${installed_nevra_old}."
        return 1
    fi
    # dnf returns 1 if no downgrade available
    if [[ "$YUMDNFCMD" == "yum" ]]; then
      # yum v3.4.3 has trouble resolving dependencies. Specify all what's installed from brew-$TASK_ID to workaround
      installed_pkgs=()
      while IFS= read -r package; do
        installed_pkgs+=("$package")
      done < <("$YUMDNFCMD" list installed | awk '/@brew-'${TASK_ID}'/ {print $1}')
      "$YUMDNFCMD" -y downgrade "$name" "${installed_pkgs[@]}"
    else
      "$YUMDNFCMD" -y downgrade "$ALLOWERASING" "$name"
    fi
}

box_out() {
    local s=("$@")
    local b=
    local w=
    for l in "${s[@]}"; do
        ((w<${#l})) && { b="$l"; w="${#l}"; }
    done
    echo -e " -${b//?/-}-\n| ${b//?/ } |"
    for l in "${s[@]}"; do
        printf '| %*s |\n' "-$w" "$l"
    done
    echo -e "| ${b//?/ } |\n-${b//?/-}-"
}

msg_prepare() {
    local action="$1" && shift
    local nevra="$1" && shift
    box_out \
        "  Preparing for $action $nevra "
}

msg_run() {
    local action="$1" && shift
    local nevra="$1" && shift
    box_out \
        "  Running $action test for $nevra "
}

# rpm -q --qf "%{NEVRA}\n" openssl
# openssl-1:1.1.0h-3.el8+7.x86_64
# rpm -q --qf "%{NEVRA}\n" dhcp-common
# dhcp-common-12:4.3.6-25.el8+7.noarch
# rpm --querytags

# NEVRA - is a new package to test.
msg_usage() {
    cat << EOF
Usage:
$PROG <options>

Options:
-s, --start=YUM_HISTORY_TRANSACTION_ID       reset system to "$YUMDNFCMD" YUM history transition before start the tests
-t, --test=TYPE                              one of: install, update, downgrade, remove
-n, --nevra=NEVRA                            package to test: name-[epoch:]version-release.arch
-r, --revertchanges                          revert changes made to system
-x, --scriptletcheck                         check output and fail if problems with scriptlets are found
-f, --rpmverify                              run "rpm --verify" to verify that what's installed matches the metadata in the RPM database
-h, --help                                   display this help and exit
EOF
}

opt_str="$@"
opt=$(getopt -n "$0" --options "hvs:t:n:r:x:f" --longoptions "help,verbose,start:,test:,nevra:,revertchanges,scriptletcheck,rpmverify" -- "$@")
eval set -- "$opt"
while [[ $# -gt 0 ]]; do
    case "$1" in
        -s|--start)
            START="$2"
            shift 2
            ;;
        -t|--test)
            TEST="$2"
            shift 2
            ;;
        -f|--rpmverify)
            RPM_VERIFY="yes"
            shift
            ;;
        -n|--nevra)
            NEVRA="$2"
            shift 2
            ;;
        -v|--verbose)
            DEBUG="yes"
            shift
            ;;
        -h|--help)
            msg_usage
            exit 0
            ;;
        -r|--revertchanges)
            REVERT="yes"
            shift
            ;;
        -x|--scriptletcheck)
            CHECK_SCRIPTLETS="yes"
            shift
            ;;
        --)
            shift
            ;;
        *)
            msg_usage
            exit 1
    esac
done

do_yum_revert() {
    rc=$?;
    trap - SIGINT SIGTERM SIGABRT EXIT # clear the trap
    echo "Revert $YUMDNFCMD history to: $HISTORY_ID"
    echo "Rollback to $YUMDNFCMD history transaction ID: $HISTORY_ID"
    "$YUMDNFCMD" -y history rollback "$HISTORY_ID"
    exit $rc
}

# Entry

: "${CHECK_SCRIPTLETS:=}"
: "${RPM_VERIFY:=}"

DEBUG_OUT="/dev/null"
if [ -n "${DEBUG:=}" ]; then
    DEBUG_OUT="/dev/stdout"
fi

debug "TEST: ${TEST:=}"
debug "DNF HISTORY START: ${START:=}"
debug "NEVRA: ${NEVRA:=}"

# Test correct invocation
if [ -z "$NEVRA" ] || ! [[ "$TEST" == "install" || "$TEST" == "update" || "$TEST" == "downgrade" || "$TEST" == "remove" ]]; then
  echo "Use: $PROG -h for help."
  exit
fi

if [ -n "${REVERT:=}" ]; then
    # Save current YUM transaction ID, to revert it later
    HISTORY_ID="$("$YUMDNFCMD" history list | sed -n -e '0,/^-\+$/d;s/^[[:space:]]*//p' | tr -s ' ' | cut -f 1 -d ' ' | sed -n -e '1p')"
    echo "Current $YUMDNFCMD history transaction ID: $HISTORY_ID"
    trap do_yum_revert SIGINT SIGTERM SIGABRT EXIT
fi

if [ -n "$START" ]; then
    echo "Revert $YUMDNFCMD history before test run to: $START"
    "$YUMDNFCMD" -y history rollback "$START"
fi

if [[ "$TEST" == "install" || "$TEST" == "remove" ]]; then
    if ! pkg_can_be_removed "$NEVRA"; then
        echo "Skipping test: $TEST for $NEVRA. Package cannot be clearly removed."
        exit $EXIT_CODE_SKIP
    fi
fi

case $TEST in
    install)
        msg_prepare "installing" "$NEVRA"
        pkg_remove_any "$NEVRA"
        msg_run "install" "$NEVRA"
        if [ -n "$CHECK_SCRIPTLETS" ]; then
            run_with_scriptlet_check pkg_install_if_absent "$NEVRA"
        else
            pkg_install_if_absent "$NEVRA"
        fi
        ret=$?
        if [[ "$ret" -eq 0 && -n "$RPM_VERIFY" ]]; then
            # The "install" test succeeded so we can proceed and rpm-verify the results
            rpm_db_verify "$NEVRA"
            ret=$?
        fi
        exit $ret
        ;;
    update)
        msg_prepare "updating" "$NEVRA"
        old_nevra="$(get_old_nevra "$NEVRA" || true)"
        if [ -z "$old_nevra" ]; then
            echo "Cannot find older package than $NEVRA. Skipping test."
            exit $EXIT_CODE_SKIP
        fi
        echo "Found available older package: $old_nevra"
        # If there's a bug preventing the old package from installing
        # the dnf downgrade fails but we can't say whether it's because
        # of the new package or the old one.
        # So rather, remove the package, install old version, and if that fails
        # skip the test.
        if pkg_can_be_removed "$NEVRA"; then
            pkg_remove_any "$NEVRA"
        else
            pkg_downgrade_if_present "$NEVRA"
        fi
        if ! pkg_install_if_absent "$old_nevra"; then
            echo "Failed to install $old_nevra. Skipping test."
            exit $EXIT_CODE_SKIP
        fi
        # At this point pkg is installed, but can be many installonlypkg(foo) packages
        pkg_remove_any_newer "$NEVRA"
        msg_run "update" "$NEVRA"
        if [ -n "$CHECK_SCRIPTLETS" ]; then
            run_with_scriptlet_check pkg_update_to_new_if_present "$NEVRA"
        else
            pkg_update_to_new_if_present "$NEVRA"
        fi
        ret=$?
        if [[ "$ret" -eq 0 && -n "$RPM_VERIFY" ]]; then
            # The "update" test succeeded so we can proceed and rpm-verify the results
            rpm_db_verify "$NEVRA"
            ret=$?
        fi
        exit $ret
        ;;
    downgrade)
        msg_prepare "downgrading" "$NEVRA"
        old_nevra="$(get_old_nevra "$NEVRA" || true)"
        if [ -z "$old_nevra" ]; then
            echo "Cannot find older package than $NEVRA. Skipping test."
            exit $EXIT_CODE_SKIP
        fi
        echo "Older available package is: $old_nevra"
        pkg_update_to_new_if_present "$NEVRA"
        if ! pkg_install_if_absent "$NEVRA"; then
            echo "Failed to install $NEVRA. Skipping test."
            exit $EXIT_CODE_SKIP
        fi
        pkg_remove_any_older "$NEVRA" # This is for installonlypkg(foo) packages, like kernel
        msg_run "downgrade" "$NEVRA"
        if [ -n "$CHECK_SCRIPTLETS" ]; then
            run_with_scriptlet_check pkg_downgrade_to_old "$NEVRA"
        else
            pkg_downgrade_to_old "$NEVRA"
        fi
        ;;
    remove)
        msg_prepare "removing" "$NEVRA"
        pkg_update_to_new_if_present "$NEVRA"
        pkg_install_if_absent "$NEVRA" || exit 1

        # OSCI-4333: We have to check again if the package can be removed
        # now after we installed its latest version. It can happen that
        # the update introduced new /etc/yum/protected.d/* file which makes
        # it uninstallable.
        if ! pkg_can_be_removed "$NEVRA"; then
            echo "Skipping test: $TEST for $NEVRA. Package cannot be clearly removed."
            exit $EXIT_CODE_SKIP
        fi

        msg_run "remove" "$NEVRA"
        if [ -n "$CHECK_SCRIPTLETS" ]; then
            run_with_scriptlet_check pkg_remove_any "$NEVRA"
        else
            pkg_remove_any "$NEVRA"
        fi
        ;;
    *)
        echo "Use: $PROG -h for help."
        exit
        ;;
esac