tests: Switch functional testing to use tmt framework

This also changes from running the unit tests, which are now run as part
of the rpm build, to running ksvalidator and ksflatten on some test
kickstarts.

./tests/good-ks/ contains kickstarts expected to pass
./tests/bad-ks/ contains kickstarts expected to fail
./tests/include-ks/ contains kickstarts expected to pass, with includes

(note that the %include directives in good-ks are not processed by
ksvalidator).

.fmf/version

@@ -0,0 +1 @@
+1

gating.yaml

@@ -1,6 +1,7 @@
 --- !Policy
 product_versions:
   - fedora-*
-decision_context: bodhi_update_push_testing
+decision_contexts: [bodhi_update_push_stable]
+subject_type: koji_build
 rules:
-  - !PassingTestCaseRule {test_case_name: dist.depcheck}
+  - !PassingTestCaseRule {test_case_name: fedora-ci.koji-build.tier0.functional}

plans/run-tests.fmf

@@ -0,0 +1,8 @@
+summary: Run pykickstart tests
+prepare:
+    how: install
+    package:
+    - python3-kickstart
+    - pykickstart
+execute:
+    script: ./tests/scripts/run_tests.sh

tests/bad-ks/fedora-disk-base.ks

@@ -0,0 +1,47 @@
+text
+lang en_US.UTF-8
+keyboard us
+timezone US/Eastern
+selinux --enforcing
+firewall --enabled --service=mdns
+services --enabled=sshd,NetworkManager,chronyd
+network --bootproto=dhcp --device=link --activate
+rootpw --lock --iscrypted locked
+shutdown
+
+bootloader --timeout=1
+
+zerombr
+clearpart --all --initlabel --disklabel=msdos
+
+# make sure that initial-setup runs and lets us do all the configuration bits
+firstboot --reconfig
+
+repo --name "packages" --url https://nowhere/
+
+%packages
+@core
+@standard
+@hardware-support
+
+kernel
+# on 32bit arm make sure we only install one kernel
+-kernel-lpae
+# remove this in %post
+dracut-config-generic
+-dracut-config-rescue
+# install tools needed to manage and boot arm systems
+@arm-tools
+chrony
+bcm283x-firmware
+initial-setup
+# Intel wireless firmware assumed never of use for disk images
+-iwl*
+-ipw*
+-usb_modeswitch
+-generic-release*
+
+# make sure all the locales are available for inital-setup and anaconda to work
+glibc-all-langpacks
+
+%end

tests/bad-ks/no-commands.ks

@@ -0,0 +1,2 @@
+echo "not a kickstart"
+without any --commands

tests/good-ks/fedora-disk-base.ks

@@ -0,0 +1,104 @@
+# fedora-disk-base.ks
+#
+# Defines the basics for all kickstarts in the fedora-live branch
+# Does not include package selection (other then mandatory)
+# Does not include localization packages or configuration
+#
+# Does includes "default" language configuration (kickstarts including
+# this template can override these settings)
+
+text
+lang en_US.UTF-8
+keyboard us
+timezone US/Eastern
+selinux --enforcing
+firewall --enabled --service=mdns
+services --enabled=sshd,NetworkManager,chronyd
+network --bootproto=dhcp --device=link --activate
+rootpw --lock --iscrypted locked
+shutdown
+
+bootloader --timeout=1
+
+zerombr
+clearpart --all --initlabel --disklabel=msdos
+
+# make sure that initial-setup runs and lets us do all the configuration bits
+firstboot --reconfig
+
+%include fedora-repo.ks
+
+%packages
+@core
+@standard
+@hardware-support
+
+kernel
+# on 32bit arm make sure we only install one kernel
+-kernel-lpae
+# remove this in %post
+dracut-config-generic
+-dracut-config-rescue
+# install tools needed to manage and boot arm systems
+@arm-tools
+chrony
+bcm283x-firmware
+initial-setup
+# Intel wireless firmware assumed never of use for disk images
+-iwl*
+-ipw*
+-usb_modeswitch
+-generic-release*
+
+# make sure all the locales are available for inital-setup and anaconda to work
+glibc-all-langpacks
+
+%end
+
+%post
+
+# Find the architecture we are on
+arch=$(uname -m)
+
+# Setup Raspberry Pi firmware
+if [[ $arch == "aarch64" ]] || [[ $arch == "armv7l" ]]; then
+if [[ $arch == "aarch64" ]]; then
+cp -P /usr/share/uboot/rpi_3/u-boot.bin /boot/efi/rpi3-u-boot.bin
+cp -P /usr/share/uboot/rpi_4/u-boot.bin /boot/efi/rpi4-u-boot.bin
+cp -P /usr/share/uboot/rpi_arm64/u-boot.bin /boot/efi/rpi-u-boot.bin
+else
+cp -P /usr/share/uboot/rpi_2/u-boot.bin /boot/efi/rpi2-u-boot.bin
+cp -P /usr/share/uboot/rpi_3_32b/u-boot.bin /boot/efi/rpi3-u-boot.bin
+cp -P /usr/share/uboot/rpi_4_32b/u-boot.bin /boot/efi/rpi4-u-boot.bin
+fi
+fi
+
+releasever=$(rpm --eval '%{fedora}')
+rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-primary
+echo "Packages within this disk image"
+rpm -qa --qf '%{size}\t%{name}-%{version}-%{release}.%{arch}\n' |sort -rn
+
+# remove random seed, the newly installed instance should make it's own
+rm -f /var/lib/systemd/random-seed
+
+# The enp1s0 interface is a left over from the imagefactory install, clean this up
+rm -f /etc/NetworkManager/system-connections/*.nmconnection
+
+dnf -y remove dracut-config-generic
+
+# Remove machine-id on pre generated images
+rm -f /etc/machine-id
+touch /etc/machine-id
+
+# Note that running rpm recreates the rpm db files which aren't needed or wanted
+rm -f /var/lib/rpm/__db*
+
+# Anaconda adds console=tty0 to the grub boot line on all images. this is problematic
+# when you are using fedora via serial console as you do not get any output post grub
+# linux does a good job of knowing what consoles need to be enabled.
+# https://bugzilla.redhat.com/show_bug.cgi?id=2022757
+if [[ $arch == "aarch64" ]] || [[ $arch == "armv7l" ]]; then
+sed -i -e 's|console=tty0||g' /boot/loader/entries/*conf
+fi
+
+%end

tests/good-ks/fedora-live-base.ks

@@ -0,0 +1,106 @@
+# fedora-live-base.ks
+#
+# Defines the basics for all kickstarts in the fedora-live branch
+# Does not include package selection (other then mandatory)
+# Does not include localization packages or configuration
+#
+# Does includes "default" language configuration (kickstarts including
+# this template can override these settings)
+
+lang en_US.UTF-8
+keyboard us
+timezone US/Eastern
+selinux --enforcing
+firewall --enabled --service=mdns
+xconfig --startxonboot
+zerombr
+clearpart --all
+part / --size 5120 --fstype ext4
+services --enabled=NetworkManager,ModemManager --disabled=sshd
+network --bootproto=dhcp --device=link --activate
+rootpw --lock --iscrypted locked
+shutdown
+
+%include fedora-repo.ks
+
+%packages
+# Explicitly specified here:
+# <notting> walters: because otherwise dependency loops cause yum issues.
+kernel
+kernel-modules
+kernel-modules-extra
+
+# The point of a live image is to install
+anaconda
+anaconda-install-env-deps
+anaconda-live
+@anaconda-tools
+# Anaconda has a weak dep on this and we don't want it on livecds, see
+# https://fedoraproject.org/wiki/Changes/RemoveDeviceMapperMultipathFromWorkstationLiveCD
+-fcoe-utils
+-device-mapper-multipath
+
+# Need aajohan-comfortaa-fonts for the SVG rnotes images
+aajohan-comfortaa-fonts
+
+# Without this, initramfs generation during live image creation fails: #1242586
+dracut-live
+
+# anaconda needs the locales available to run for different locales
+glibc-all-langpacks
+
+# provide the livesys scripts
+livesys-scripts
+%end
+
+%post
+# Enable livesys services
+systemctl enable livesys.service
+systemctl enable livesys-late.service
+
+# enable tmpfs for /tmp
+systemctl enable tmp.mount
+
+# make it so that we don't do writing to the overlay for things which
+# are just tmpdirs/caches
+# note https://bugzilla.redhat.com/show_bug.cgi?id=1135475
+cat >> /etc/fstab << EOF
+vartmp   /var/tmp    tmpfs   defaults   0  0
+EOF
+
+# work around for poor key import UI in PackageKit
+rm -f /var/lib/rpm/__db*
+echo "Packages within this LiveCD"
+rpm -qa --qf '%{size}\t%{name}-%{version}-%{release}.%{arch}\n' |sort -rn
+# Note that running rpm recreates the rpm db files which aren't needed or wanted
+rm -f /var/lib/rpm/__db*
+
+# go ahead and pre-make the man -k cache (#455968)
+/usr/bin/mandb
+
+# make sure there aren't core files lying around
+rm -f /core*
+
+# remove random seed, the newly installed instance should make it's own
+rm -f /var/lib/systemd/random-seed
+
+# convince readahead not to collect
+# FIXME: for systemd
+
+echo 'File created by kickstart. See systemd-update-done.service(8).' \
+    | tee /etc/.updated >/var/.updated
+
+# Drop the rescue kernel and initramfs, we don't need them on the live media itself.
+# See bug 1317709
+rm -f /boot/*-rescue*
+
+# Disable network service here, as doing it in the services line
+# fails due to RHBZ #1369794
+systemctl disable network
+
+# Remove machine-id on pre generated images
+rm -f /etc/machine-id
+touch /etc/machine-id
+
+%end
+

tests/include-ks/fedora-live-base.ks

@@ -0,0 +1,106 @@
+# fedora-live-base.ks
+#
+# Defines the basics for all kickstarts in the fedora-live branch
+# Does not include package selection (other then mandatory)
+# Does not include localization packages or configuration
+#
+# Does includes "default" language configuration (kickstarts including
+# this template can override these settings)
+
+lang en_US.UTF-8
+keyboard us
+timezone US/Eastern
+selinux --enforcing
+firewall --enabled --service=mdns
+xconfig --startxonboot
+zerombr
+clearpart --all
+##part / --size 5120 --fstype ext4
+services --enabled=NetworkManager,ModemManager --disabled=sshd
+network --bootproto=dhcp --device=link --activate
+rootpw --lock --iscrypted locked
+shutdown
+
+%include fedora-repo.ks
+
+%packages
+# Explicitly specified here:
+# <notting> walters: because otherwise dependency loops cause yum issues.
+kernel
+kernel-modules
+kernel-modules-extra
+
+# The point of a live image is to install
+anaconda
+anaconda-install-env-deps
+anaconda-live
+@anaconda-tools
+# Anaconda has a weak dep on this and we don't want it on livecds, see
+# https://fedoraproject.org/wiki/Changes/RemoveDeviceMapperMultipathFromWorkstationLiveCD
+-fcoe-utils
+-device-mapper-multipath
+
+# Need aajohan-comfortaa-fonts for the SVG rnotes images
+aajohan-comfortaa-fonts
+
+# Without this, initramfs generation during live image creation fails: #1242586
+dracut-live
+
+# anaconda needs the locales available to run for different locales
+glibc-all-langpacks
+
+# provide the livesys scripts
+livesys-scripts
+%end
+
+%post
+# Enable livesys services
+systemctl enable livesys.service
+systemctl enable livesys-late.service
+
+# enable tmpfs for /tmp
+systemctl enable tmp.mount
+
+# make it so that we don't do writing to the overlay for things which
+# are just tmpdirs/caches
+# note https://bugzilla.redhat.com/show_bug.cgi?id=1135475
+cat >> /etc/fstab << EOF
+vartmp   /var/tmp    tmpfs   defaults   0  0
+EOF
+
+# work around for poor key import UI in PackageKit
+rm -f /var/lib/rpm/__db*
+echo "Packages within this LiveCD"
+rpm -qa --qf '%{size}\t%{name}-%{version}-%{release}.%{arch}\n' |sort -rn
+# Note that running rpm recreates the rpm db files which aren't needed or wanted
+rm -f /var/lib/rpm/__db*
+
+# go ahead and pre-make the man -k cache (#455968)
+/usr/bin/mandb
+
+# make sure there aren't core files lying around
+rm -f /core*
+
+# remove random seed, the newly installed instance should make it's own
+rm -f /var/lib/systemd/random-seed
+
+# convince readahead not to collect
+# FIXME: for systemd
+
+echo 'File created by kickstart. See systemd-update-done.service(8).' \
+    | tee /etc/.updated >/var/.updated
+
+# Drop the rescue kernel and initramfs, we don't need them on the live media itself.
+# See bug 1317709
+rm -f /boot/*-rescue*
+
+# Disable network service here, as doing it in the services line
+# fails due to RHBZ #1369794
+systemctl disable network
+
+# Remove machine-id on pre generated images
+rm -f /etc/machine-id
+touch /etc/machine-id
+
+%end
+

tests/include-ks/fedora-live-minimization.ks

@@ -0,0 +1,9 @@
+# Common packages removed from comps
+# For F14, these removals should be moved to comps itself
+
+%packages
+
+# save some space
+-hplip
+
+%end

tests/include-ks/fedora-live-xfce.ks

@@ -0,0 +1,33 @@
+# fedora-livecd-xfce.ks
+#
+# Description:
+# - Fedora Live Spin with the light-weight XFCE Desktop Environment
+#
+# Maintainer(s):
+# - Rahul Sundaram    <sundaram@fedoraproject.org>
+# - Christoph Wickert <cwickert@fedoraproject.org>
+# - Kevin Fenzi       <kevin@tummy.com>
+# - Adam Miller       <maxamillion@fedoraproject.org>
+
+%include fedora-live-base.ks
+%include fedora-live-minimization.ks
+%include fedora-xfce-common.ks
+
+# need a bigger /
+part / --size 6144
+
+%post
+# xfce configuration
+
+# create /etc/sysconfig/desktop (needed for installation)
+
+cat > /etc/sysconfig/desktop <<EOF
+PREFERRED=/usr/bin/startxfce4
+DISPLAYMANAGER=/usr/sbin/lightdm
+EOF
+
+# set livesys session type
+sed -i 's/^livesys_session=.*/livesys_session="xfce"/' /etc/sysconfig/livesys
+
+%end
+

tests/include-ks/fedora-repo-rawhide.ks

@@ -0,0 +1,2 @@
+repo --name=rawhide --mirrorlist=https://mirrors.fedoraproject.org/mirrorlist?repo=rawhide&arch=$basearch
+url --mirrorlist=https://mirrors.fedoraproject.org/mirrorlist?repo=rawhide&arch=$basearch

tests/include-ks/fedora-repo.ks

@@ -0,0 +1,9 @@
+# Include the appropriate repo definitions
+
+# Exactly one of the following should be uncommented
+
+# For the master branch the following should be uncommented
+%include fedora-repo-rawhide.ks
+
+# For non-master branches the following should be uncommented
+# %include fedora-repo-not-rawhide.ks

tests/include-ks/fedora-xfce-common.ks

@@ -0,0 +1,34 @@
+# fedora-livecd-xfce.ks
+#
+# Description:
+# - Fedora Live Spin with the light-weight XFCE Desktop Environment
+#
+# Maintainer(s):
+# - Kevin Fenzi       <kevin@tummy.com>
+# - Adam Miller       <maxamillion@fedoraproject.org>
+# - Mukundan Ragavan  <nonamedotc@fedoraproject.org>
+
+%packages
+
+fedora-release-xfce
+# install env-group to resolve RhBug:1891500
+@^xfce-desktop-environment
+
+@xfce-apps
+@xfce-extra-plugins
+@xfce-media
+@xfce-office
+
+wget
+system-config-printer
+
+# save some space
+-autofs
+-acpid
+-gimp-help
+-desktop-backgrounds-basic
+-aspell-*                   # dictionaries are big
+-xfce4-sensors-plugin
+-xfce4-eyes-plugin
+
+%end

tests/scripts/run_tests.sh

@@ -1,31 +1,24 @@
 #!/bin/sh
 
-PATH=/usr/bin
-TMPDIR="$(mktemp -d)"
-CWD="$(pwd)"
+echo "Running pykickstart tests"
+cd ./tests/
 
-PACKAGE=pykickstart
-BRANCH=
+# Run ksvalidator on good kickstart examples
+for f in ./good-ks/*.ks; do
+    echo "Checking $f"
+    ksvalidator $f || exit 1
+done
 
-# clone the dist-git tree for this package
-cd ${TMPDIR}
-fedpkg co ${PACKAGE}
-cd ${PACKAGE}
-[ -z "${BRANCH}" ] || fedpkg switch-branch ${BRANCH}
-fedpkg prep
+# Run ksvalidator on bad kickstart examples
+for f in ./bad-ks/*.ks; do
+    echo "Checking $f"
+    ksvalidator $f && exit 1
+done
 
-# scramble together the extracted source tree name
-SRCDIR="${PACKAGE}-$(grep Version: ${PACKAGE}.spec | cut -d ' ' -f 2)"
 
-# run the tests
-cd ${SRCDIR}
-make check
-CHECK_RET=$?
-make test
-TEST_RET=$?
-RET=$((${CHECK_RET} + ${TEST_RET}))
+# Run ksflatten on a set of kickstarts
+echo "Testing ksflatten with included kickstarts"
+ksflatten -c ./include-ks/fedora-live-xfce.ks -o flat.ks || exit 1
+ksvalidator flat.ks || exit 1
 
-# clean up and exit
-cd ${CWD}
-rm -rf ${TMPDIR}
-exit ${RET}
+exit 0