root_image

#!/usr/bin/env bash
#
# Create a VM image suitable for running automated tests
# Output: vm_image

set -o nounset
set -o errexit
set -o errtrace

ktest_dir=$(dirname "$(readlink -f "$0")")
debootstrap=$ktest_dir/debootstrap/debootstrap

. "$ktest_dir/lib/util.sh"
. "$ktest_dir/lib/common.sh"

if [[ $(id -u) != 0 ]] ; then
    echo this script must be run as root
    exit 1
fi

checkdep fallocate util-linux
checkdep mkfs.ext4 e2fsprogs
checkdep curl 

IMAGE_SIZE="10G"
MIRROR=https://deb.debian.org/debian/

usage()
{
    echo "root_image: create/update virtual machine root images for ktest"
    echo "Usage: root_image cmd [options]"
    echo "  create		Create a new image"
    echo "  update		Update an existing image"
    echo "  sync		Copy directory to an existing image"
    echo
    echo "options:"
    echo "  -h			Display this help and exit"
    echo "  -a <arch>		Architecture for vm image"
    echo "  -m <mirror>		Debian mirror"
    echo '  -i <image>		Image to create/update, defaults to /var/lib/ktest/root.$arch'
}

if [[ $# = 0 ]]; then
    usage
    exit 1
fi

ktest_image=""
CMD="cmd_$1"
shift

while getopts "ha:m:i:" arg; do
    case $arg in
	h)
	    usage
	    exit 0
	    ;;
	a)
	    ktest_arch=$OPTARG
	    ;;
	m)
	    MIRROR=$OPTARG
	    ;;
	i)
	    ktest_image=$OPTARG
	    ;;
    esac
done
shift $(( OPTIND - 1 ))

parse_arch "$ktest_arch"

[[ -z $ktest_image ]] && ktest_image=/var/lib/ktest/root.$DEBIAN_ARCH

mkdir -p "$(dirname "$ktest_image")"

PACKAGES=(kexec-tools less psmisc openssh-server curl		\
    pciutils							\
    pkg-config libtool-bin					\
    gdb strace linux-perf trace-cmd blktrace sysstat iotop htop	\
    hdparm mdadm lvm2						\
    btrfs-progs jfsutils nilfs-tools f2fs-tools			\
    bc attr gawk acl rsync git python3-docutils			\
    stress-ng lsof xxd)

# build tools:
PACKAGES+=(build-essential make gcc g++ clang)
PACKAGES+=(autoconf automake autopoint bison)
PACKAGES+=(cargo)
PACKAGES+=(devscripts debhelper)

# stress testing:
PACKAGES+=(fio dbench bonnie++ fsmark)

# bcachefs-tools build dependencies:
PACKAGES+=(libblkid-dev uuid-dev libscrypt-dev libsodium-dev)
PACKAGES+=(libkeyutils-dev liburcu-dev libudev-dev zlib1g-dev libattr1-dev systemd-dev)
PACKAGES+=(libaio-dev libzstd-dev liblz4-dev libfuse3-dev valgrind)
PACKAGES+=(llvm libclang-dev)

# quota tools:
PACKAGES+=(libudev-dev libldap2-dev)

# xfstests:
PACKAGES+=(acct bsdextrautils xfsprogs xfslibs-dev quota libcap2-bin)
PACKAGES+=(libattr1-dev libaio-dev libgdbm-dev libacl1-dev gettext)
PACKAGES+=(libssl-dev libgdbm-dev libgdbm-compat-dev liburing-dev)
PACKAGES+=(duperemove fsverity)

# xfsprogs:
PACKAGES+=(libinih-dev)

# bcachefs:
# Not currently packaged by debian:
#PACKAGES+=(bcachefs-tools)

# nfs testing:
PACKAGES+=(nfs-kernel-server)

# ocfs2 testing:
PACKAGES+=(ocfs2-tools)

# afs
PACKAGES+=(openafs-dbserver openafs-fileserver)

# nbd testing
PACKAGES+=(nbd-client nbd-server)

# dm testing:
PACKAGES+=(cryptsetup)

# weird block layer crap
PACKAGES+=(multipath-tools sg3-utils srptools)

# PuzzleFS support
PACKAGES+=(capnproto jq)

PACKAGES+=(erofs-utils squashfs-tools-ng)

# ZFS support
#PACKAGES+=("linux-headers-generic" dkms zfsutils-linux zfs-dkms)

# Lustre support
PACKAGES+=(libyaml-0-2 libyaml-dev)

# suspend testing:
# [[ $KERNEL_ARCH = x86 ]] && PACKAGES+=(uswsusp)

EXCLUDE=(dmidecode nano rsyslog logrotate cron		\
    iptables nfacct					\
    debconf-i18n info gnupg libpam-systemd)

SYSTEMD_MASK=(dev-hvc0.device				\
    getty.target					\
    getty-static.service				\
    avahi-daemon.service				\
    crond.service					\
    exim4.service					\
    kdump.service					\
    hdparm.service					\
    cdrom.mount						\
    mdadm-raid.service					\
    lvm2-activation-early.service			\
    aoetools.service					\
    sysstat.service					\
    kexec-load.service					\
    kexec.service					\
    systemd-ask-password-console.path			\
    systemd-ask-password-wall.path			\
    systemd-update-utmp-runlevel.service		\
    systemd-update-utmp.service				\
    time-sync.target					\
    multipathd.service)

export DEBIAN_FRONTEND=noninteractive
export DEBCONF_NONINTERACTIVE_SEEN=true
export LC_ALL=C
export LANGUAGE=C
export LANG=C
# We compute it here to avoid on hosts systems (e.g. NixOS)
# that does not possess `chroot` in the isolated `PATH`
# to fail miserably.
export CHROOT=$(which chroot)

_chroot()
{
    PATH=/usr/sbin:/usr/bin:/sbin:/bin "$CHROOT" "$@"
}

update_files()
{
    install -m0644 "$ktest_dir/lib/fstab" "$MNT/etc/fstab"
    install -m0755 "$ktest_dir/lib/testrunner.wrapper" "$MNT/sbin/testrunner.wrapper"
    install -m0644 "$ktest_dir/lib/testrunner.service" "$MNT/lib/systemd/system/testrunner.service"

    ln -sf /lib/systemd/system/testrunner.service "$MNT/etc/systemd/system/multi-user.target.wants/testrunner.service"

    touch "$MNT/etc/resolv.conf"
    chmod 644 "$MNT/etc/resolv.conf"

    mkdir -p "$MNT/root/"
    install -m0644 "$MNT/etc/skel/.bashrc" "$MNT/root/"
    install -m0644 "$MNT/etc/skel/.profile" "$MNT/root/"

    cat >> "$MNT/root/.bashrc" <<-ZZ
    export PROMPT_COMMAND="history -a; \$PROMPT_COMMAND"
ZZ

    mkdir -p "$MNT/var/log/core"
    chmod 777 "$MNT/var/log/core"

    # Disable systemd/udev stuff we don't need:

    # systemctl mask doesn't work for foreign archs
    #_chroot "$MNT" systemctl mask "${SYSTEMD_MASK[@]}"

    for i in "${SYSTEMD_MASK[@]}"; do
	(cd "$MNT/etc/systemd/system"; ln -sf /dev/null "$i")
    done

    cat > "$MNT/etc/systemd/journald.conf" <<-ZZ
[Journal]
Storage=none
ForwardToConsole=no
MaxLevelConsole=emerg
ZZ

    mkdir -p "$MNT/etc/network"
    cat > "$MNT/etc/network/interfaces" <<-ZZ
auto lo
iface lo inet loopback

auto eth0
iface eth0 inet dhcp
ZZ

    # disable network interface renaming - it's unreliable
    mkdir -p "$MNT/etc/udev/rules.d/"
    ln -sf /dev/null "$MNT/etc/udev/rules.d/80-net-setup-link.rules"

    rm -f "$MNT/lib/udev/rules.d/*persistent*"
    rm -f "$MNT/lib/udev/rules.d/*lvm*"
    rm -f "$MNT/lib/udev/rules.d/*dm*"
    rm -f "$MNT/lib/udev/rules.d/*md-raid*"
    rm -f "$MNT/lib/udev/rules.d/*btrfs*"
    rm -f "$MNT/lib/udev/rules.d/*hdparm*"

    echo $(hostname)-kvm > "$MNT/etc/hostname"

    ln -sf /ktest-out/.bash_history "$MNT/root/.bash_history"
}

update_packages()
{
    # systemd... !?
    mkdir -p "$MNT"/run/user/0
    cp /etc/resolv.conf "$MNT/etc/resolv.conf"

    mkdir -p "$MNT"/etc/dpkg/dpkg.cfg.d/
    echo force-unsafe-io > "$MNT"/etc/dpkg/dpkg.cfg.d/dpkg-unsafe

    _chroot "$MNT" mount -t proc none /proc
    _chroot "$MNT" apt-get -qq update
    _chroot "$MNT" apt-get -qq upgrade
    _chroot "$MNT" apt-get -qq install --no-install-recommends "${PACKAGES[@]}"
    rm -f "$MNT/var/cache/apt/archives/*.deb"
}

_umount_image()
{
    umount --recursive "$MNT"
    rmdir "$MNT"
    trap '' EXIT
}

umount_image()
{
    _umount_image

    e2fsck -f "$ktest_image".new
    resize2fs -M "$ktest_image".new		# shrinks the file
    resize2fs "$ktest_image".new "$IMAGE_SIZE"	# re-grows as sparse
    mv "$ktest_image".new "$ktest_image"
}

mount_image()
{
    MNT=$(mktemp --tmpdir -d $(basename "$0")-XXXXXXXXXX)
    trap '_umount_image; rm -f "$ktest_image".new' EXIT

    cp "$ktest_image" "$ktest_image".new
    mount "$ktest_image".new "$MNT"
}

cmd_update()
{
    if [[ ! -e $ktest_image ]]; then
	echo "$ktest_image does not exist"
	exit 1
    fi

    mount_image
    update_packages
    update_files
    umount_image
}

cmd_update_files()
{
    if [[ ! -e $ktest_image ]]; then
	echo "$ktest_image does not exist"
	exit 1
    fi

    mount_image
    update_files
    umount_image
}

cmd_sync()
{
    local source="${1:-$(pwd)/}"
    local source_dir="$(basename "$source")"

    if [[ ! -e "$ktest_image" ]]; then
	echo "$ktest_image does not exist"
	exit 1
    fi

    mount_image
    rm -rf "$MNT/workspace/$source_dir"
    mkdir -p "$MNT/workspace/$source_dir"
    rsync --archive -r "$source" "$MNT/workspace/$source_dir"
    umount_image
}

cmd_init()
{
    (cd "$ktest_dir"; git submodule update --init debootstrap)
}

cmd_create()
{
    if ! [[ -f $debootstrap ]]; then
	echo "Run root_image init to prep debootstrap"
	exit 1
    fi

    if [[ -e $ktest_image ]]; then
	echo "$ktest_image already exists"
	exit 1
    fi

    MNT=$(mktemp --tmpdir -d $(basename "$0")-XXXXXXXXXX)
    trap '_umount_image; rm -f "$ktest_image".new' EXIT

    fallocate -l "$IMAGE_SIZE" "$ktest_image".new
    chmod 644 "$ktest_image".new
    mkfs.ext4 -F "$ktest_image".new
    mount "$ktest_image".new "$MNT"

    mkdir -p "$MNT"/etc/dpkg/dpkg.cfg.d/
    echo force-unsafe-io > "$MNT"/etc/dpkg/dpkg.cfg.d/dpkg-unsafe

    DEBOOTSTRAP_DIR=$ktest_dir/debootstrap $debootstrap	\
	--no-check-gpg					\
	--arch="$DEBIAN_ARCH"				\
	--exclude=$(join_by , "${EXCLUDE[@]}")		\
	--no-merged-usr					\
	--foreign					\
	sid "$MNT" "$MIRROR"
    mkdir -p "$MNT"/etc/dpkg/dpkg.cfg.d/
    echo force-unsafe-io > "$MNT"/etc/dpkg/dpkg.cfg.d/dpkg-unsafe

    _chroot "$MNT" /debootstrap/debootstrap --second-stage
    _chroot "$MNT" dpkg --configure -a

    update_packages
    update_files
    umount_image
}

if [[ $(type -t "$CMD") != function ]]; then
    usage
    exit 1
fi

$CMD "$@"