diff --git a/tests/README b/tests/README new file mode 100644 index 0000000..672acdb --- /dev/null +++ b/tests/README @@ -0,0 +1,25 @@ +I'm sorry. The playbooks here are a much-too-complicated way of saying: + + - test podman (root and rootless) under cgroups v2 + - reboot into cgroups v1 + - repeat the same podman tests + +We can't use standard-test-basic any more because, tl;dr, that has to +be the last stanza in the playbook and it doesn't offer any mechanism +for running a reboot in the middle of tests. (I actually found a way +but it was even uglier than this approach). + +The starting point is tests.yml . From there: + + tests.yml + \- test_podman.yml + |- roles/rootless_user_ready/ + \- test_podman_cgroups_vn.yml (runs twice: cgroups v2, v1) + |- roles/set_cgroups/ + \- roles/run_bats_tests/ (runs tests: root, rootless) + +Principal result is the file 'artifacts/test.log'. It will contain +one line for each test run, format will be '(PASS|FAIL|ERROR) ' + +For each completed test there will also be a 'test..bats.log' +containing some setup blurbs (RPMs, environment) and the full BATS log. diff --git a/tests/check_results.yml b/tests/check_results.yml new file mode 100644 index 0000000..e873b8a --- /dev/null +++ b/tests/check_results.yml @@ -0,0 +1,21 @@ +--- +# Copied from standard-test-basic +- name: check results + local_action: shell egrep "^(FAIL|ERROR)" "{{ artifacts }}/test.log" + register: test_fails + # Never fail at this step. Just store result of tests. + failed_when: False + +- name: preserve results + set_fact: + role_result_failed: "{{ (test_fails.stdout|d|length > 0) or (test_fails.stderr|d|length > 0) }}" + role_result_msg: "{{ test_fails.stdout|d('tests failed.') }}" + +- name: display results + vars: + msg: | + Tests failed: {{ role_result_failed|d('Undefined') }} + Tests msg: {{ role_result_msg|d('None') }} + debug: + msg: "{{ msg.split('\n') }}" + failed_when: "role_result_failed|bool" diff --git a/tests/roles/rootless_user_ready/tasks/main.yml b/tests/roles/rootless_user_ready/tasks/main.yml new file mode 100644 index 0000000..cfed03b --- /dev/null +++ b/tests/roles/rootless_user_ready/tasks/main.yml @@ -0,0 +1,6 @@ +--- +- name: make sure rootless account exists + user: name={{ rootless_user }} + +- name: rootless account | enable linger + shell: loginctl enable-linger {{ rootless_user }} diff --git a/tests/roles/run_bats_tests/files/run_bats_tests.sh b/tests/roles/run_bats_tests/files/run_bats_tests.sh new file mode 100755 index 0000000..17061a0 --- /dev/null +++ b/tests/roles/run_bats_tests/files/run_bats_tests.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# +# Run bats tests for a given $TEST_PACKAGE, e.g. buildah, podman +# +# This is invoked by the 'run_bats_tests' role; we assume that +# the package foo has a foo-tests subpackage which provides the +# directory /usr/share/foo/test/system, containing one or more .bats +# test files. +# +# We create two files: +# +# /tmp/test.summary.log - one-liner with FAIL, PASS, ERROR and a blurb +# /tmp/test.bats.log - full log of this script, plus the BATS run +# +export PATH=/usr/local/bin:/usr/sbin:/usr/bin + +FULL_LOG=/tmp/test.bats.log +rm -f $FULL_LOG +touch $FULL_LOG + +# Preserve output to a log file, but also emit on stdout. This covers +# RHEL (which preserves logfiles but runs ansible without --verbose) +# and Fedora (which hides logfiles but runs ansible --verbose). +exec &> >(tee -a $FULL_LOG) + +# Log program versions +echo "Packages:" +rpm -qa |\ + egrep 'podman|conmon|crun|runc|iptable|slirp|systemd|container-selinux' |\ + sort |\ + sed -e 's/^/ /' + +divider='------------------------------------------------------------------' +echo $divider +printenv | sort +echo $divider + +testdir=/usr/share/${TEST_PACKAGE}/test/system + +if ! cd $testdir; then + echo "FAIL ${TEST_NAME} : cd $testdir" > /tmp/test.summary.log + exit 0 +fi + +echo "\$ bats ." +bats . +rc=$? + +echo $divider +echo "bats completed with status $rc" + +status=PASS +if [ $rc -ne 0 ]; then + status=FAIL +fi + +echo "${status} ${TEST_NAME}" > /tmp/test.summary.log + +# FIXME: for CI purposes, always exit 0. This allows subsequent tests. +exit 0 diff --git a/tests/roles/run_bats_tests/tasks/main.yml b/tests/roles/run_bats_tests/tasks/main.yml new file mode 100644 index 0000000..a207c05 --- /dev/null +++ b/tests/roles/run_bats_tests/tasks/main.yml @@ -0,0 +1,10 @@ +--- +# Create empty results file, world-writable +- name: initialize test.log file + copy: dest=/tmp/test.log content='' force=yes mode=0666 + +- name: execute tests + include: run_one_test.yml + with_items: "{{ tests }}" + loop_control: + loop_var: test diff --git a/tests/roles/run_bats_tests/tasks/run_one_test.yml b/tests/roles/run_bats_tests/tasks/run_one_test.yml new file mode 100644 index 0000000..2451606 --- /dev/null +++ b/tests/roles/run_bats_tests/tasks/run_one_test.yml @@ -0,0 +1,69 @@ +--- +- name: "{{ test.name }} | install test packages" + dnf: name="{{ test.package }}-tests" state=installed + +- name: "{{ test.name }} | define helper variables" + set_fact: + test_name_oneword: "{{ test.name | replace(' ','-') }}" + +# UGH. This is necessary because our caller sets some environment variables +# and we need to set a few more based on other caller variables; then we +# need to combine the two dicts when running the test. This seems to be +# the only way to do it in ansible. +- name: "{{ test.name }} | define local environment" + set_fact: + local_environment: + TEST_NAME: "{{ test.name }}" + TEST_PACKAGE: "{{ test.package }}" + TEST_ENV: "{{ test.environment }}" + +- name: "{{ test.name }} | setup/teardown helper | see if exists" + local_action: stat path={{ role_path }}/files/helper.{{ test_name_oneword }}.sh + register: helper + +- name: "{{ test.name }} | setup/teardown helper | install" + copy: src=helper.{{ test_name_oneword }}.sh dest=/tmp/helper.sh + when: helper.stat.exists + +# This is what runs the BATS tests. +- name: "{{ test.name }} | run test" + script: ./run_bats_tests.sh + args: + chdir: /usr/share/{{ test.package }}/test/system + become: "{{ true if test.become is defined else false }}" + become_user: "{{ rootless_user }}" + environment: "{{ local_environment | combine(test.environment) }}" + +# BATS tests will always exit zero and should leave behind two files: +# a full log (test.bats.log) and a one-line PASS/FAIL file (.summary.log) +- name: "{{ test.name }} | pull logs" + fetch: + src: "/tmp/test.{{ item }}.log" + dest: "{{ artifacts }}/test.{{ test_name_oneword }}.{{ item }}.log" + flat: yes + with_items: + - bats + - summary + +# Collect all the one-line PASS/FAIL results in one file, test.log +- name: "{{ test.name }} | keep running tally of test results" + local_action: shell cat "{{ artifacts }}/test.{{ test_name_oneword }}.summary.log" >>"{{ artifacts }}/test.log" + +# ...then delete the oneliner file, to keep things clean. +- name: "{{ test.name }} | clean up status file" + local_action: file dest="{{ artifacts }}/test.{{ test_name_oneword }}.summary.log" state=absent + +# FIXME: do the same with results.yml: +# - test: {{ test.name }} +# result: pass or fail +# logs: +# - test.{{ test_name_oneword }}.bats.log + +- name: "{{ test.name }} | remove remote logs and helpers" + file: + dest=/tmp/{{ item }} + state=absent + with_items: + - test.bats.log + - test.summary.log + - helper.sh diff --git a/tests/roles/set_cgroups/tasks/main.yml b/tests/roles/set_cgroups/tasks/main.yml new file mode 100644 index 0000000..8843e49 --- /dev/null +++ b/tests/roles/set_cgroups/tasks/main.yml @@ -0,0 +1,61 @@ +--- +# Check the CURRENT cgroup level; we get this from /proc/cmdline +- name: check current kernel options + shell: fgrep systemd.unified_cgroup_hierarchy=0 /proc/cmdline + register: result + ignore_errors: true + +- name: determine current cgroups | assume v2 + set_fact: current_cgroups=2 + +- name: determine current cgroups | looks like v1 + set_fact: current_cgroups=1 + when: result is succeeded + +- debug: + msg: "want: v{{ want_cgroups }} actual: v{{ current_cgroups }}" + +# Update grubenv file to reflect the desired cgroup level +- name: remove cgroup option from kernel flags + shell: + cmd: sed -i -e "s/^\(kernelopts=.*\)systemd\.unified_cgroup_hierarchy=.\(.*\)/\1\2/" /boot/grub2/grubenv + warn: false + +- name: add it with the desired value + shell: + cmd: sed -i -e "s/^\(kernelopts=.*\)/\1 systemd.unified_cgroup_hierarchy=0/" /boot/grub2/grubenv + warn: false + when: want_cgroups == 1 + +# If want != have, reboot +- name: reboot + reboot: + reboot_timeout: 900 + register: back_again + ignore_errors: yes + when: want_cgroups|int != current_cgroups|int + +- name: wait and reconnect + wait_for_connection: + timeout: 900 + register: back_again2 + when: want_cgroups|int != current_cgroups|int and not back_again.changed + +- set_fact: + expected_fstype: + - none + - tmpfs + - cgroup2fs + +- name: confirm cgroups setting + shell: stat -f -c "%T" /sys/fs/cgroup + register: fstype + +- debug: + msg: "stat(/sys/fs/cgroup) = {{ fstype.stdout }}" + +- name: system cgroups is the expected type + assert: + that: + - fstype.stdout == expected_fstype[want_cgroups|int] + fail_msg: "stat(/sys/fs/cgroup) = {{ fstype.stdout }} (expected {{ expected_fstype[want_cgroups|int] }})" diff --git a/tests/test_podman.sh b/tests/test_podman.sh deleted file mode 100755 index 75bcc53..0000000 --- a/tests/test_podman.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -e -# -# Simple podman tests -# - -# Log program and kernel versions -echo "Important package versions:" -( - uname -r - rpm -qa | egrep 'podman|conmon|crun|runc|iptable|slirp|systemd|container-selinux' | sort -) | sed -e 's/^/ /' - -# Log environment; or at least the useful bits -echo "Environment:" -env | grep -v LS_COLORS= | sort | sed -e 's/^/ /' - -bats /usr/share/podman/test/system diff --git a/tests/test_podman.yml b/tests/test_podman.yml index 7002972..27927de 100644 --- a/tests/test_podman.yml +++ b/tests/test_podman.yml @@ -1,22 +1,32 @@ --- - hosts: localhost + tags: + - classic + - container + vars: + - artifacts: ./artifacts + rootless_user: testuser roles: - - role: standard-test-basic - tags: - - classic - - container - required_packages: - - bats - - podman - - podman-tests - tests: - - root-test: - dir: ./ - run: ./test_podman.sh - timeout: 15m - - rootless-test: - # running the test with su doesn't create the directory for fedora user on /run/user/ - # so create it manually - dir: ./ - run: loginctl enable-linger fedora; su -c ${PWD}/test_podman.sh - fedora - timeout: 15m + - role: rootless_user_ready + + tasks: + # At the start of a run, clean up state. Useful for test reruns. + - name: local artifacts directory exists + local_action: file path="{{ artifacts }}" state=directory + + - name: remove stale log files + local_action: shell rm -f {{ artifacts }}/test*.log + + - name: clear test results + local_action: command truncate --size=0 {{ artifacts }}/test.log + + # These are the actual tests: set cgroups vN, then run root/rootless tests. + - name: set cgroups and run podman tests + include_tasks: test_podman_cgroups_vn.yml + loop: [ 2, 1 ] + loop_control: + loop_var: want_cgroups + + # Postprocessing: check for FAIL or ERROR in any test, exit 1 if so + - name: check results + include_tasks: check_results.yml diff --git a/tests/test_podman_cgroups_vn.yml b/tests/test_podman_cgroups_vn.yml new file mode 100644 index 0000000..f069182 --- /dev/null +++ b/tests/test_podman_cgroups_vn.yml @@ -0,0 +1,19 @@ +--- +# Requires: 'want_cgroups' variable set to 1 or 2 +- include_role: + name: set_cgroups +- include_role: + name: run_bats_tests + vars: + tests: + # Yes, this is horrible duplication, but trying to refactor in ansible + # yields even more horrible unreadable code. This is the lesser evil. + - name: podman root cgroupsv{{ want_cgroups }} + package: podman + environment: + PODMAN: /usr/bin/podman + - name: podman rootless cgroupsv{{ want_cgroups }} + package: podman + environment: + PODMAN: /usr/bin/podman + become: true