From 977477db9d58b6bd595b75e9f4520f921ade92e0 Mon Sep 17 00:00:00 2001 From: "Brian C. Lane" Date: Tue, 21 Feb 2023 11:27:31 -0800 Subject: [PATCH] tests: Switch functional testing to use tmt framework This also changes from running the unit tests, which are now run as part of the rpm build, to running ksvalidator and ksflatten on some test kickstarts. ./tests/good-ks/ contains kickstarts expected to pass ./tests/bad-ks/ contains kickstarts expected to fail ./tests/include-ks/ contains kickstarts expected to pass, with includes (note that the %include directives in good-ks are not processed by ksvalidator). --- .fmf/version | 1 + gating.yaml | 5 +- plans/run-tests.fmf | 8 ++ tests/bad-ks/fedora-disk-base.ks | 47 ++++++++ tests/bad-ks/no-commands.ks | 2 + tests/good-ks/fedora-disk-base.ks | 104 ++++++++++++++++++ tests/good-ks/fedora-live-base.ks | 106 +++++++++++++++++++ tests/include-ks/fedora-live-base.ks | 106 +++++++++++++++++++ tests/include-ks/fedora-live-minimization.ks | 9 ++ tests/include-ks/fedora-live-xfce.ks | 33 ++++++ tests/include-ks/fedora-repo-rawhide.ks | 2 + tests/include-ks/fedora-repo.ks | 9 ++ tests/include-ks/fedora-xfce-common.ks | 34 ++++++ tests/scripts/run_tests.sh | 41 +++---- 14 files changed, 481 insertions(+), 26 deletions(-) create mode 100644 .fmf/version create mode 100644 plans/run-tests.fmf create mode 100644 tests/bad-ks/fedora-disk-base.ks create mode 100644 tests/bad-ks/no-commands.ks create mode 100644 tests/good-ks/fedora-disk-base.ks create mode 100644 tests/good-ks/fedora-live-base.ks create mode 100644 tests/include-ks/fedora-live-base.ks create mode 100644 tests/include-ks/fedora-live-minimization.ks create mode 100644 tests/include-ks/fedora-live-xfce.ks create mode 100644 tests/include-ks/fedora-repo-rawhide.ks create mode 100644 tests/include-ks/fedora-repo.ks create mode 100644 tests/include-ks/fedora-xfce-common.ks diff --git a/.fmf/version b/.fmf/version new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/.fmf/version @@ -0,0 +1 @@ +1 diff --git a/gating.yaml b/gating.yaml index d71584a..0c1cc35 100644 --- a/gating.yaml +++ b/gating.yaml @@ -1,6 +1,7 @@ --- !Policy product_versions: - fedora-* -decision_context: bodhi_update_push_testing +decision_contexts: [bodhi_update_push_stable] +subject_type: koji_build rules: - - !PassingTestCaseRule {test_case_name: dist.depcheck} + - !PassingTestCaseRule {test_case_name: fedora-ci.koji-build.tier0.functional} diff --git a/plans/run-tests.fmf b/plans/run-tests.fmf new file mode 100644 index 0000000..be27fa6 --- /dev/null +++ b/plans/run-tests.fmf @@ -0,0 +1,8 @@ +summary: Run pykickstart tests +prepare: + how: install + package: + - python3-kickstart + - pykickstart +execute: + script: ./tests/scripts/run_tests.sh diff --git a/tests/bad-ks/fedora-disk-base.ks b/tests/bad-ks/fedora-disk-base.ks new file mode 100644 index 0000000..5f4a9eb --- /dev/null +++ b/tests/bad-ks/fedora-disk-base.ks @@ -0,0 +1,47 @@ +text +lang en_US.UTF-8 +keyboard us +timezone US/Eastern +selinux --enforcing +firewall --enabled --service=mdns +services --enabled=sshd,NetworkManager,chronyd +network --bootproto=dhcp --device=link --activate +rootpw --lock --iscrypted locked +shutdown + +bootloader --timeout=1 + +zerombr +clearpart --all --initlabel --disklabel=msdos + +# make sure that initial-setup runs and lets us do all the configuration bits +firstboot --reconfig + +repo --name "packages" --url https://nowhere/ + +%packages +@core +@standard +@hardware-support + +kernel +# on 32bit arm make sure we only install one kernel +-kernel-lpae +# remove this in %post +dracut-config-generic +-dracut-config-rescue +# install tools needed to manage and boot arm systems +@arm-tools +chrony +bcm283x-firmware +initial-setup +# Intel wireless firmware assumed never of use for disk images +-iwl* +-ipw* +-usb_modeswitch +-generic-release* + +# make sure all the locales are available for inital-setup and anaconda to work +glibc-all-langpacks + +%end diff --git a/tests/bad-ks/no-commands.ks b/tests/bad-ks/no-commands.ks new file mode 100644 index 0000000..d50b27b --- /dev/null +++ b/tests/bad-ks/no-commands.ks @@ -0,0 +1,2 @@ +echo "not a kickstart" +without any --commands diff --git a/tests/good-ks/fedora-disk-base.ks b/tests/good-ks/fedora-disk-base.ks new file mode 100644 index 0000000..ea67746 --- /dev/null +++ b/tests/good-ks/fedora-disk-base.ks @@ -0,0 +1,104 @@ +# fedora-disk-base.ks +# +# Defines the basics for all kickstarts in the fedora-live branch +# Does not include package selection (other then mandatory) +# Does not include localization packages or configuration +# +# Does includes "default" language configuration (kickstarts including +# this template can override these settings) + +text +lang en_US.UTF-8 +keyboard us +timezone US/Eastern +selinux --enforcing +firewall --enabled --service=mdns +services --enabled=sshd,NetworkManager,chronyd +network --bootproto=dhcp --device=link --activate +rootpw --lock --iscrypted locked +shutdown + +bootloader --timeout=1 + +zerombr +clearpart --all --initlabel --disklabel=msdos + +# make sure that initial-setup runs and lets us do all the configuration bits +firstboot --reconfig + +%include fedora-repo.ks + +%packages +@core +@standard +@hardware-support + +kernel +# on 32bit arm make sure we only install one kernel +-kernel-lpae +# remove this in %post +dracut-config-generic +-dracut-config-rescue +# install tools needed to manage and boot arm systems +@arm-tools +chrony +bcm283x-firmware +initial-setup +# Intel wireless firmware assumed never of use for disk images +-iwl* +-ipw* +-usb_modeswitch +-generic-release* + +# make sure all the locales are available for inital-setup and anaconda to work +glibc-all-langpacks + +%end + +%post + +# Find the architecture we are on +arch=$(uname -m) + +# Setup Raspberry Pi firmware +if [[ $arch == "aarch64" ]] || [[ $arch == "armv7l" ]]; then +if [[ $arch == "aarch64" ]]; then +cp -P /usr/share/uboot/rpi_3/u-boot.bin /boot/efi/rpi3-u-boot.bin +cp -P /usr/share/uboot/rpi_4/u-boot.bin /boot/efi/rpi4-u-boot.bin +cp -P /usr/share/uboot/rpi_arm64/u-boot.bin /boot/efi/rpi-u-boot.bin +else +cp -P /usr/share/uboot/rpi_2/u-boot.bin /boot/efi/rpi2-u-boot.bin +cp -P /usr/share/uboot/rpi_3_32b/u-boot.bin /boot/efi/rpi3-u-boot.bin +cp -P /usr/share/uboot/rpi_4_32b/u-boot.bin /boot/efi/rpi4-u-boot.bin +fi +fi + +releasever=$(rpm --eval '%{fedora}') +rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-primary +echo "Packages within this disk image" +rpm -qa --qf '%{size}\t%{name}-%{version}-%{release}.%{arch}\n' |sort -rn + +# remove random seed, the newly installed instance should make it's own +rm -f /var/lib/systemd/random-seed + +# The enp1s0 interface is a left over from the imagefactory install, clean this up +rm -f /etc/NetworkManager/system-connections/*.nmconnection + +dnf -y remove dracut-config-generic + +# Remove machine-id on pre generated images +rm -f /etc/machine-id +touch /etc/machine-id + +# Note that running rpm recreates the rpm db files which aren't needed or wanted +rm -f /var/lib/rpm/__db* + +# Anaconda adds console=tty0 to the grub boot line on all images. this is problematic +# when you are using fedora via serial console as you do not get any output post grub +# linux does a good job of knowing what consoles need to be enabled. +# https://bugzilla.redhat.com/show_bug.cgi?id=2022757 +if [[ $arch == "aarch64" ]] || [[ $arch == "armv7l" ]]; then +sed -i -e 's|console=tty0||g' /boot/loader/entries/*conf +fi + +%end diff --git a/tests/good-ks/fedora-live-base.ks b/tests/good-ks/fedora-live-base.ks new file mode 100644 index 0000000..205e114 --- /dev/null +++ b/tests/good-ks/fedora-live-base.ks @@ -0,0 +1,106 @@ +# fedora-live-base.ks +# +# Defines the basics for all kickstarts in the fedora-live branch +# Does not include package selection (other then mandatory) +# Does not include localization packages or configuration +# +# Does includes "default" language configuration (kickstarts including +# this template can override these settings) + +lang en_US.UTF-8 +keyboard us +timezone US/Eastern +selinux --enforcing +firewall --enabled --service=mdns +xconfig --startxonboot +zerombr +clearpart --all +part / --size 5120 --fstype ext4 +services --enabled=NetworkManager,ModemManager --disabled=sshd +network --bootproto=dhcp --device=link --activate +rootpw --lock --iscrypted locked +shutdown + +%include fedora-repo.ks + +%packages +# Explicitly specified here: +# walters: because otherwise dependency loops cause yum issues. +kernel +kernel-modules +kernel-modules-extra + +# The point of a live image is to install +anaconda +anaconda-install-env-deps +anaconda-live +@anaconda-tools +# Anaconda has a weak dep on this and we don't want it on livecds, see +# https://fedoraproject.org/wiki/Changes/RemoveDeviceMapperMultipathFromWorkstationLiveCD +-fcoe-utils +-device-mapper-multipath + +# Need aajohan-comfortaa-fonts for the SVG rnotes images +aajohan-comfortaa-fonts + +# Without this, initramfs generation during live image creation fails: #1242586 +dracut-live + +# anaconda needs the locales available to run for different locales +glibc-all-langpacks + +# provide the livesys scripts +livesys-scripts +%end + +%post +# Enable livesys services +systemctl enable livesys.service +systemctl enable livesys-late.service + +# enable tmpfs for /tmp +systemctl enable tmp.mount + +# make it so that we don't do writing to the overlay for things which +# are just tmpdirs/caches +# note https://bugzilla.redhat.com/show_bug.cgi?id=1135475 +cat >> /etc/fstab << EOF +vartmp /var/tmp tmpfs defaults 0 0 +EOF + +# work around for poor key import UI in PackageKit +rm -f /var/lib/rpm/__db* +echo "Packages within this LiveCD" +rpm -qa --qf '%{size}\t%{name}-%{version}-%{release}.%{arch}\n' |sort -rn +# Note that running rpm recreates the rpm db files which aren't needed or wanted +rm -f /var/lib/rpm/__db* + +# go ahead and pre-make the man -k cache (#455968) +/usr/bin/mandb + +# make sure there aren't core files lying around +rm -f /core* + +# remove random seed, the newly installed instance should make it's own +rm -f /var/lib/systemd/random-seed + +# convince readahead not to collect +# FIXME: for systemd + +echo 'File created by kickstart. See systemd-update-done.service(8).' \ + | tee /etc/.updated >/var/.updated + +# Drop the rescue kernel and initramfs, we don't need them on the live media itself. +# See bug 1317709 +rm -f /boot/*-rescue* + +# Disable network service here, as doing it in the services line +# fails due to RHBZ #1369794 +systemctl disable network + +# Remove machine-id on pre generated images +rm -f /etc/machine-id +touch /etc/machine-id + +%end + diff --git a/tests/include-ks/fedora-live-base.ks b/tests/include-ks/fedora-live-base.ks new file mode 100644 index 0000000..bf89758 --- /dev/null +++ b/tests/include-ks/fedora-live-base.ks @@ -0,0 +1,106 @@ +# fedora-live-base.ks +# +# Defines the basics for all kickstarts in the fedora-live branch +# Does not include package selection (other then mandatory) +# Does not include localization packages or configuration +# +# Does includes "default" language configuration (kickstarts including +# this template can override these settings) + +lang en_US.UTF-8 +keyboard us +timezone US/Eastern +selinux --enforcing +firewall --enabled --service=mdns +xconfig --startxonboot +zerombr +clearpart --all +##part / --size 5120 --fstype ext4 +services --enabled=NetworkManager,ModemManager --disabled=sshd +network --bootproto=dhcp --device=link --activate +rootpw --lock --iscrypted locked +shutdown + +%include fedora-repo.ks + +%packages +# Explicitly specified here: +# walters: because otherwise dependency loops cause yum issues. +kernel +kernel-modules +kernel-modules-extra + +# The point of a live image is to install +anaconda +anaconda-install-env-deps +anaconda-live +@anaconda-tools +# Anaconda has a weak dep on this and we don't want it on livecds, see +# https://fedoraproject.org/wiki/Changes/RemoveDeviceMapperMultipathFromWorkstationLiveCD +-fcoe-utils +-device-mapper-multipath + +# Need aajohan-comfortaa-fonts for the SVG rnotes images +aajohan-comfortaa-fonts + +# Without this, initramfs generation during live image creation fails: #1242586 +dracut-live + +# anaconda needs the locales available to run for different locales +glibc-all-langpacks + +# provide the livesys scripts +livesys-scripts +%end + +%post +# Enable livesys services +systemctl enable livesys.service +systemctl enable livesys-late.service + +# enable tmpfs for /tmp +systemctl enable tmp.mount + +# make it so that we don't do writing to the overlay for things which +# are just tmpdirs/caches +# note https://bugzilla.redhat.com/show_bug.cgi?id=1135475 +cat >> /etc/fstab << EOF +vartmp /var/tmp tmpfs defaults 0 0 +EOF + +# work around for poor key import UI in PackageKit +rm -f /var/lib/rpm/__db* +echo "Packages within this LiveCD" +rpm -qa --qf '%{size}\t%{name}-%{version}-%{release}.%{arch}\n' |sort -rn +# Note that running rpm recreates the rpm db files which aren't needed or wanted +rm -f /var/lib/rpm/__db* + +# go ahead and pre-make the man -k cache (#455968) +/usr/bin/mandb + +# make sure there aren't core files lying around +rm -f /core* + +# remove random seed, the newly installed instance should make it's own +rm -f /var/lib/systemd/random-seed + +# convince readahead not to collect +# FIXME: for systemd + +echo 'File created by kickstart. See systemd-update-done.service(8).' \ + | tee /etc/.updated >/var/.updated + +# Drop the rescue kernel and initramfs, we don't need them on the live media itself. +# See bug 1317709 +rm -f /boot/*-rescue* + +# Disable network service here, as doing it in the services line +# fails due to RHBZ #1369794 +systemctl disable network + +# Remove machine-id on pre generated images +rm -f /etc/machine-id +touch /etc/machine-id + +%end + diff --git a/tests/include-ks/fedora-live-minimization.ks b/tests/include-ks/fedora-live-minimization.ks new file mode 100644 index 0000000..3cde144 --- /dev/null +++ b/tests/include-ks/fedora-live-minimization.ks @@ -0,0 +1,9 @@ +# Common packages removed from comps +# For F14, these removals should be moved to comps itself + +%packages + +# save some space +-hplip + +%end diff --git a/tests/include-ks/fedora-live-xfce.ks b/tests/include-ks/fedora-live-xfce.ks new file mode 100644 index 0000000..1506163 --- /dev/null +++ b/tests/include-ks/fedora-live-xfce.ks @@ -0,0 +1,33 @@ +# fedora-livecd-xfce.ks +# +# Description: +# - Fedora Live Spin with the light-weight XFCE Desktop Environment +# +# Maintainer(s): +# - Rahul Sundaram +# - Christoph Wickert +# - Kevin Fenzi +# - Adam Miller + +%include fedora-live-base.ks +%include fedora-live-minimization.ks +%include fedora-xfce-common.ks + +# need a bigger / +part / --size 6144 + +%post +# xfce configuration + +# create /etc/sysconfig/desktop (needed for installation) + +cat > /etc/sysconfig/desktop < +# - Adam Miller +# - Mukundan Ragavan + +%packages + +fedora-release-xfce +# install env-group to resolve RhBug:1891500 +@^xfce-desktop-environment + +@xfce-apps +@xfce-extra-plugins +@xfce-media +@xfce-office + +wget +system-config-printer + +# save some space +-autofs +-acpid +-gimp-help +-desktop-backgrounds-basic +-aspell-* # dictionaries are big +-xfce4-sensors-plugin +-xfce4-eyes-plugin + +%end diff --git a/tests/scripts/run_tests.sh b/tests/scripts/run_tests.sh index fdb2fd5..5f5f3dc 100755 --- a/tests/scripts/run_tests.sh +++ b/tests/scripts/run_tests.sh @@ -1,31 +1,24 @@ #!/bin/sh -PATH=/usr/bin -TMPDIR="$(mktemp -d)" -CWD="$(pwd)" +echo "Running pykickstart tests" +cd ./tests/ -PACKAGE=pykickstart -BRANCH= +# Run ksvalidator on good kickstart examples +for f in ./good-ks/*.ks; do + echo "Checking $f" + ksvalidator $f || exit 1 +done -# clone the dist-git tree for this package -cd ${TMPDIR} -fedpkg co ${PACKAGE} -cd ${PACKAGE} -[ -z "${BRANCH}" ] || fedpkg switch-branch ${BRANCH} -fedpkg prep +# Run ksvalidator on bad kickstart examples +for f in ./bad-ks/*.ks; do + echo "Checking $f" + ksvalidator $f && exit 1 +done -# scramble together the extracted source tree name -SRCDIR="${PACKAGE}-$(grep Version: ${PACKAGE}.spec | cut -d ' ' -f 2)" -# run the tests -cd ${SRCDIR} -make check -CHECK_RET=$? -make test -TEST_RET=$? -RET=$((${CHECK_RET} + ${TEST_RET})) +# Run ksflatten on a set of kickstarts +echo "Testing ksflatten with included kickstarts" +ksflatten -c ./include-ks/fedora-live-xfce.ks -o flat.ks || exit 1 +ksvalidator flat.ks || exit 1 -# clean up and exit -cd ${CWD} -rm -rf ${TMPDIR} -exit ${RET} +exit 0