From 2457f22baf3187351448f0c5c5d5d9bce432827d Mon Sep 17 00:00:00 2001 From: Kairui Song Date: Fri, 31 Jul 2020 15:34:39 +0800 Subject: [PATCH] selftest: Add basic infrastructure to build test image The Makefile In tests/ could help build a VM image using Fedora cloud image as base image, or, user can specify a base image using BASE_IMAGE=. The current repo will be packeged and installed in the image, so the image could be used as a test image to test kexec-tools. The image building is splited into two steps: The first step, it either convert the base image to qcow2 or create a snapshot on it, and install basic packages (dracut, grubby, ...) and do basic setups (setup crashkernel=, disable selinux, ...). See tests/scripts/build-scripts/base-image.sh for detail. The second step, it creates a snapshot on top of the image produced by the previous step, and install the packaged kexec-tools of current repo. See tests/scripts/build-scripts/test-base-image.sh for detail. In this way, if repo's content is changes, `make` will detect it and only rebuild the second snapshot which speed up the rebuild by a lot. The image will be located as tests/output/test-base-image, and in qcow2 format. And default user/password is set to root/fedora. Signed-off-by: Kairui Song Acked-by: Dave Young --- tests/Makefile | 80 ++++++ tests/scripts/build-image.sh | 57 ++++ tests/scripts/build-scripts/base-image.sh | 11 + .../scripts/build-scripts/test-base-image.sh | 15 ++ tests/scripts/image-init-lib.sh | 245 ++++++++++++++++++ 5 files changed, 408 insertions(+) create mode 100644 tests/Makefile create mode 100755 tests/scripts/build-image.sh create mode 100755 tests/scripts/build-scripts/base-image.sh create mode 100755 tests/scripts/build-scripts/test-base-image.sh create mode 100644 tests/scripts/image-init-lib.sh diff --git a/tests/Makefile b/tests/Makefile new file mode 100644 index 0000000..a352ab3 --- /dev/null +++ b/tests/Makefile @@ -0,0 +1,80 @@ +BASE_IMAGE ?= + +TEST_ROOT := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) +BUILD_ROOT := $(TEST_ROOT)/build +REPO = $(shell realpath $(TEST_ROOT)/../) +ARCH ?= $(shell arch) +SPEC = kexec-tools.spec + +DIST ?= fedora +DIST_ABR ?= f +DIST_ABRL ?= fc +DIST_UNSET ?= rhel +RELEASE ?= 32 + +DEFAULT_BASE_IMAGE_VER ?= 1.6 +DEFAULT_BASE_IMAGE ?= Fedora-Cloud-Base-$(RELEASE)-$(DEFAULT_BASE_IMAGE_VER).$(ARCH).raw.xz +DEFAULT_BASE_IMAGE_URL ?= https://dl.fedoraproject.org/pub/fedora/linux/releases/$(RELEASE)/Cloud/$(ARCH)/images/$(DEFAULT_BASE_IMAGE) + +BUILD_ROOT = $(TEST_ROOT)/build +RPMDEFINE = --define '_sourcedir $(REPO)'\ + --define '_specdir $(REPO)'\ + --define '_builddir $(BUILD_ROOT)'\ + --define '_srcrpmdir $(BUILD_ROOT)'\ + --define '_rpmdir $(BUILD_ROOT)'\ + --define 'dist %{?distprefix}.$(DIST_ABRL)$(RELEASE)'\ + --define '$(DIST) $(RELEASE)'\ + --eval '%undefine $(DIST_UNSET)'\ + --define '$(DIST_ABRL)$(RELEASE) 1'\ + +KEXEC_TOOLS_SRC = $(filter-out $(REPO)/tests,$(wildcard $(REPO)/*)) +KEXEC_TOOLS_TEST_SRC = $(wildcard $(REPO)/tests/scripts/**/*) +KEXEC_TOOLS_NVR = $(shell rpm $(RPMDEFINE) -q --specfile $(REPO)/$(SPEC) 2>/dev/null | grep -m 1 .) +KEXEC_TOOLS_RPM = $(BUILD_ROOT)/$(ARCH)/$(KEXEC_TOOLS_NVR).rpm + +all: $(TEST_ROOT)/output/test-base-image + +# Use either: +# fedpkg --release $(DIST_ABR)$(RELEASE) --path ../../ local +# or +# rpmbuild $(RPMDEFINE) -ba $(REPO)/$(SPEC) +# to rebuild the rpm, currently use rpmbuild to have better control over the rpm building process +# +$(KEXEC_TOOLS_RPM): $(KEXEC_TOOLS_SRC) + sh -c "cd .. && fedpkg sources" + @echo Rebuilding RPM due to modification of sources: $? + rpmbuild $(RPMDEFINE) -ba $(REPO)/$(SPEC) + +$(BUILD_ROOT)/base-image: + mkdir -p $(BUILD_ROOT) +ifeq ($(strip $(BASE_IMAGE)),) + wget $(DEFAULT_BASE_IMAGE_URL) -O $(BUILD_ROOT)/$(DEFAULT_BASE_IMAGE) + $(TEST_ROOT)/scripts/build-image.sh \ + $(BUILD_ROOT)/$(DEFAULT_BASE_IMAGE)\ + $(BUILD_ROOT)/base-image +else + $(TEST_ROOT)/scripts/build-image.sh \ + $(BASE_IMAGE)\ + $(BUILD_ROOT)/base-image +endif + +$(BUILD_ROOT)/inst-base-image: $(BUILD_ROOT)/base-image + @echo "Building installation base image" + echo $(KEXEC_TOOLS_NVR) + $(TEST_ROOT)/scripts/build-image.sh \ + $(BUILD_ROOT)/base-image \ + $(BUILD_ROOT)/inst-base-image \ + $(TEST_ROOT)/scripts/build-scripts/base-image.sh + +$(TEST_ROOT)/output/test-base-image: $(BUILD_ROOT)/inst-base-image $(KEXEC_TOOLS_RPM) $(KEXEC_TOOLS_TEST_SRC) + @echo "Building test base image" + mkdir -p $(TEST_ROOT)/output + $(TEST_ROOT)/scripts/build-image.sh \ + $(BUILD_ROOT)/inst-base-image \ + $(TEST_ROOT)/output/test-base-image \ + $(TEST_ROOT)/scripts/build-scripts/test-base-image.sh \ + $(KEXEC_TOOLS_RPM) + +clean: + rm -rf $(TEST_ROOT)/build + rm -rf $(TEST_ROOT)/output diff --git a/tests/scripts/build-image.sh b/tests/scripts/build-image.sh new file mode 100755 index 0000000..c196bfb --- /dev/null +++ b/tests/scripts/build-image.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +if [ $# -lt 2 ]; then + echo "Usage: $(basename $0) [] + Build a new on top of , and install + contents defined in . are directly passed + to . + + If is raw, will copy it and create + in qcow2 format. + + If is qcow2, will create as a snapshot + on top of " + exit 1 +fi + +BASEDIR=$(realpath $(dirname "$0")) +. $BASEDIR/image-init-lib.sh + +# Base image to build from +BASE_IMAGE=$1 && shift +if [[ ! -e $BASE_IMAGE ]]; then + perror_exit "Base image '$BASE_IMAGE' not found" +else + BASE_IMAGE=$(realpath "$BASE_IMAGE") +fi + +OUTPUT_IMAGE=$1 && shift +if [[ ! -d $(dirname $OUTPUT_IMAGE) ]]; then + perror_exit "Path '$(dirname $OUTPUT_IMAGE)' doesn't exists" +fi + +INST_SCRIPT=$1 && shift + +create_image_from_base_image $BASE_IMAGE $OUTPUT_IMAGE.building + +mount_image $OUTPUT_IMAGE.building + +img_inst() { + inst_in_image $OUTPUT_IMAGE.building $@ +} + +img_inst_pkg() { + inst_pkg_in_image $OUTPUT_IMAGE.building $@ +} + +img_run_cmd() { + run_in_image $OUTPUT_IMAGE.building "$@" +} + +img_add_qemu_cmd() { + QEMU_CMD+="$@" +} + +[ -e "$INST_SCRIPT" ] && source $INST_SCRIPT + +mv $OUTPUT_IMAGE.building $OUTPUT_IMAGE diff --git a/tests/scripts/build-scripts/base-image.sh b/tests/scripts/build-scripts/base-image.sh new file mode 100755 index 0000000..59f4574 --- /dev/null +++ b/tests/scripts/build-scripts/base-image.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +img_inst_pkg grubby\ + dnsmasq\ + openssh openssh-server\ + dracut-network dracut-squash squashfs-tools ethtool snappy + +img_run_cmd "grubby --args systemd.journald.forward_to_console=1 systemd.log_target=console --update-kernel ALL" +img_run_cmd "grubby --args selinux=0 --update-kernel ALL" +img_run_cmd "grubby --args crashkernel=224M --update-kernel ALL" +img_run_cmd "mkdir -p /kexec-kdump-test" diff --git a/tests/scripts/build-scripts/test-base-image.sh b/tests/scripts/build-scripts/test-base-image.sh new file mode 100755 index 0000000..a649306 --- /dev/null +++ b/tests/scripts/build-scripts/test-base-image.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +# Test RPMs to be installed +TEST_RPMS= +for _rpm in $@; do + if [[ ! -e $_rpm ]]; then + perror_exit "'$_rpm' not found" + else + TEST_RPMS=$(realpath "$_rpm") + fi +done + +img_inst_pkg $TEST_RPMS +# Test script should start kdump manually to save time +img_run_cmd "systemctl disable kdump.service" diff --git a/tests/scripts/image-init-lib.sh b/tests/scripts/image-init-lib.sh new file mode 100644 index 0000000..372e5e4 --- /dev/null +++ b/tests/scripts/image-init-lib.sh @@ -0,0 +1,245 @@ +#!/usr/bin/env bash +[ -z "$TESTDIR" ] && TESTDIR=$(realpath $(dirname "$0")/../) + +SUDO="sudo" + +declare -A MNTS=() +declare -A DEVS=() + +perror() { + echo $@>&2 +} + +perror_exit() { + echo $@>&2 + exit 1 +} + +is_mounted() +{ + findmnt -k -n $1 &>/dev/null +} + +clean_up() +{ + for _mnt in ${MNTS[@]}; do + is_mounted $_mnt && $SUDO umount -f $_mnt + done + + for _dev in ${DEVS[@]}; do + [ ! -e "$_dev" ] && continue + [[ "$_dev" == "/dev/loop"* ]] && $SUDO losetup -d "$_dev" + [[ "$_dev" == "/dev/nbd"* ]] && $SUDO qemu-nbd --disconnect "$_dev" + done + + [ -d "$TMPDIR" ] && $SUDO rm --one-file-system -rf -- "$TMPDIR"; + + sync +} + +trap ' +ret=$?; +clean_up +exit $ret; +' EXIT + +# clean up after ourselves no matter how we die. +trap 'exit 1;' SIGINT + +readonly TMPDIR="$(mktemp -d -t kexec-kdump-test.XXXXXX)" +[ -d "$TMPDIR" ] || perror_exit "mktemp failed." + +get_image_fmt() { + local image=$1 fmt + + [ ! -e "$image" ] && perror "image: $image doesn't exist" && return 1 + + fmt=$(qemu-img info $image | sed -n "s/file format:\s*\(.*\)/\1/p") + + [ $? -eq 0 ] && echo $fmt && return 0 + + return 1 +} + +# If it's partitioned, return the mountable partition, else return the dev +get_mountable_dev() { + local dev=$1 parts + + $SUDO partprobe $dev && sync + parts="$(ls -1 ${dev}p*)" + if [ -n "$parts" ]; then + if [ $(echo "$parts" | wc -l) -gt 1 ]; then + perror "It's a image with multiple partitions, using last partition as main partition" + fi + echo "$parts" | tail -1 + else + echo "$dev" + fi +} + +prepare_loop() { + [ -n "$(lsmod | grep "^loop")" ] && return + + $SUDO modprobe loop + + [ ! -e "/dev/loop-control" ] && perror_exit "failed to load loop driver" +} + +prepare_nbd() { + [ -n "$(lsmod | grep "^nbd")" ] && return + + $SUDO modprobe nbd max_part=4 + + [ ! -e "/dev/nbd0" ] && perror_exit "failed to load nbd driver" +} + +mount_nbd() { + local image=$1 size dev + for _dev in /sys/class/block/nbd* ; do + size=$(cat $_dev/size) + if [ "$size" -eq 0 ] ; then + dev=/dev/${_dev##*/} + $SUDO qemu-nbd --connect=$dev $image 1>&2 + [ $? -eq 0 ] && echo $dev && break + fi + done + + return 1 +} + +image_lock() +{ + local image=$1 timeout=5 fd + + eval "exec {fd}>$image.lock" + if [ $? -ne 0 ]; then + perror_exit "failed acquiring image lock" + exit 1 + fi + + flock -n $fd + rc=$? + while [ $rc -ne 0 ]; do + echo "Another instance is holding the image lock ..." + flock -w $timeout $fd + rc=$? + done +} + +# Mount a device, will umount it automatially when shell exits +mount_image() { + local image=$1 fmt + local dev mnt mnt_dev + + # Lock the image just in case user run this script in parrel + image_lock $image + + fmt=$(get_image_fmt $image) + [ $? -ne 0 ] || [ -z "$fmt" ] && perror_exit "failed to detect image format" + + if [ "$fmt" == "raw" ]; then + prepare_loop + + dev="$($SUDO losetup --show -f $image)" + [ $? -ne 0 ] || [ -z "$dev" ] && perror_exit "failed to setup loop device" + + elif [ "$fmt" == "qcow2" ]; then + prepare_nbd + + dev=$(mount_nbd $image) + [ $? -ne 0 ] || [ -z "$dev" ] perror_exit "failed to connect qemu to nbd device '$dev'" + else + perror_exit "Unrecognized image format '$fmt'" + fi + DEVS[$image]="$dev" + + mnt="$(mktemp -d -p $TMPDIR -t mount.XXXXXX)" + [ $? -ne 0 ] || [ -z "$mnt" ] && perror_exit "failed to create tmp mount dir" + MNTS[$image]="$mnt" + + mnt_dev=$(get_mountable_dev "$dev") + [ $? -ne 0 ] || [ -z "$mnt_dev" ] && perror_exit "failed to setup loop device" + + $SUDO mount $mnt_dev $mnt + [ $? -ne 0 ] && perror_exit "failed to mount device '$mnt_dev'" +} + +shell_in_image() { + local image=$1 && shift + local root=${MNTS[$image]} + + pushd $root + + $SHELL + + popd +} + +inst_pkg_in_image() { + local image=$1 && shift + local root=${MNTS[$image]} + + # LSB not available + # release_info=$($SUDO chroot $root /bin/bash -c "lsb_release -a") + # release=$(echo "$release_info" | sed -n "s/Release:\s*\(.*\)/\1/p") + # distro=$(echo "$release_info" | sed -n "s/Distributor ID:\s*\(.*\)/\1/p") + # if [ "$distro" != "Fedora" ]; then + # perror_exit "only Fedora image is supported" + # fi + release=$(cat $root/etc/fedora-release | sed -n "s/.*[Rr]elease\s*\([0-9]*\).*/\1/p") + [ $? -ne 0 ] || [ -z "$release" ] && perror_exit "only Fedora image is supported" + + $SUDO dnf --releasever=$release --installroot=$root install -y $@ +} + +run_in_image() { + local image=$1 && shift + local root=${MNTS[$image]} + + echo $SUDO chroot $root /bin/bash -c $@ > /dev/stderr + $SUDO chroot $root /bin/bash -c "$@" +} + +inst_in_image() { + local image=$1 src=$2 dst=$3 + local root=${MNTS[$image]} + + $SUDO cp $src $root/$dst +} + +# If source image is qcow2, create a snapshot +# If source image is raw, convert to raw +# If source image is xz, decompress then repeat the above logic +# +# Won't touch source image +create_image_from_base_image() { + local image=$1 + local output=$2 + local decompressed_image + + local ext="${image##*.}" + if [[ "$ext" == 'xz' ]]; then + echo "Decompressing base image..." + xz -d -k $image + decompressed_image=${image%.xz} + image=$decompressed_image + fi + + local image_fmt=$(qemu-img info $image | sed -n "s/file format:\s*\(.*\)/\1/p") + if [ "$image_fmt" != "raw" ]; then + if [ "$image_fmt" == "qcow2" ]; then + echo "Source image is qcow2, using snapshot..." + qemu-img create -f qcow2 -b $image $output + else + perror_exit "Unrecognized base image format $image_mnt" + fi + else + echo "Source image is raw, converting to qcow2..." + qemu-img convert -f raw -O qcow2 $image $output + fi + + # Clean up decompress temp image + if [ -n "$decompressed_image" ]; then + rm $decompressed_image + fi +}