Complete rewrite of gating tests

This makes a mess of the gating test sources, all
for the sake of being able to run cgroups v1 tests.

Signed-off-by: Ed Santiago <santiago@redhat.com>

tests/README

@@ -0,0 +1,25 @@
+I'm sorry. The playbooks here are a much-too-complicated way of saying:
+
+   - test podman (root and rootless) under cgroups v2
+   - reboot into cgroups v1
+   - repeat the same podman tests
+
+We can't use standard-test-basic any more because, tl;dr, that has to
+be the last stanza in the playbook and it doesn't offer any mechanism
+for running a reboot in the middle of tests. (I actually found a way
+but it was even uglier than this approach).
+
+The starting point is tests.yml . From there:
+
+    tests.yml
+      \- test_podman.yml
+           |- roles/rootless_user_ready/
+           \- test_podman_cgroups_vn.yml    (runs twice: cgroups v2, v1)
+                |- roles/set_cgroups/
+                \- roles/run_bats_tests/    (runs tests: root, rootless)
+
+Principal result is the file 'artifacts/test.log'. It will contain
+one line for each test run, format will be '(PASS|FAIL|ERROR) <test name>'
+
+For each completed test there will also be a 'test.<name>.bats.log'
+containing some setup blurbs (RPMs, environment) and the full BATS log.

tests/check_results.yml

@@ -0,0 +1,21 @@
+---
+# Copied from standard-test-basic
+- name: check results
+  local_action: shell egrep "^(FAIL|ERROR)" "{{ artifacts }}/test.log"
+  register: test_fails
+  # Never fail at this step. Just store result of tests.
+  failed_when: False
+
+- name: preserve results
+  set_fact:
+    role_result_failed: "{{ (test_fails.stdout|d|length > 0) or (test_fails.stderr|d|length > 0) }}"
+    role_result_msg: "{{ test_fails.stdout|d('tests failed.') }}"
+
+- name: display results
+  vars:
+    msg: |
+       Tests failed: {{ role_result_failed|d('Undefined') }}
+       Tests msg: {{ role_result_msg|d('None') }}
+  debug:
+    msg: "{{ msg.split('\n') }}"
+  failed_when: "role_result_failed|bool"

tests/roles/rootless_user_ready/tasks/main.yml

@@ -0,0 +1,6 @@
+---
+- name: make sure rootless account exists
+  user: name={{ rootless_user }}
+
+- name: rootless account | enable linger
+  shell: loginctl enable-linger {{ rootless_user }}

tests/roles/run_bats_tests/files/run_bats_tests.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+#
+# Run bats tests for a given $TEST_PACKAGE, e.g. buildah, podman
+#
+# This is invoked by the 'run_bats_tests' role; we assume that
+# the package foo has a foo-tests subpackage which provides the
+# directory /usr/share/foo/test/system, containing one or more .bats
+# test files.
+#
+# We create two files:
+#
+#    /tmp/test.summary.log - one-liner with FAIL, PASS, ERROR and a blurb
+#    /tmp/test.bats.log    - full log of this script, plus the BATS run
+#
+export PATH=/usr/local/bin:/usr/sbin:/usr/bin
+
+FULL_LOG=/tmp/test.bats.log
+rm -f $FULL_LOG
+touch $FULL_LOG
+
+# Preserve output to a log file, but also emit on stdout. This covers
+# RHEL (which preserves logfiles but runs ansible without --verbose)
+# and Fedora (which hides logfiles but runs ansible --verbose).
+exec &> >(tee -a $FULL_LOG)
+
+# Log program versions
+echo "Packages:"
+rpm -qa |\
+    egrep 'podman|conmon|crun|runc|iptable|slirp|systemd|container-selinux' |\
+    sort |\
+    sed -e 's/^/  /'
+
+divider='------------------------------------------------------------------'
+echo $divider
+printenv | sort
+echo $divider
+
+testdir=/usr/share/${TEST_PACKAGE}/test/system
+
+if ! cd $testdir; then
+    echo "FAIL ${TEST_NAME} : cd $testdir"      > /tmp/test.summary.log
+    exit 0
+fi
+
+echo "\$ bats ."
+bats .
+rc=$?
+
+echo $divider
+echo "bats completed with status $rc"
+
+status=PASS
+if [ $rc -ne 0 ]; then
+    status=FAIL
+fi
+
+echo "${status} ${TEST_NAME}" > /tmp/test.summary.log
+
+# FIXME: for CI purposes, always exit 0. This allows subsequent tests.
+exit 0

tests/roles/run_bats_tests/tasks/main.yml

@@ -0,0 +1,10 @@
+---
+# Create empty results file, world-writable
+- name: initialize test.log file
+  copy: dest=/tmp/test.log content='' force=yes mode=0666
+
+- name: execute tests
+  include: run_one_test.yml
+  with_items: "{{ tests }}"
+  loop_control:
+    loop_var: test

tests/roles/run_bats_tests/tasks/run_one_test.yml

@@ -0,0 +1,69 @@
+---
+- name: "{{ test.name }} | install test packages"
+  dnf: name="{{ test.package }}-tests" state=installed
+
+- name: "{{ test.name }} | define helper variables"
+  set_fact:
+    test_name_oneword: "{{ test.name | replace(' ','-') }}"
+
+# UGH. This is necessary because our caller sets some environment variables
+# and we need to set a few more based on other caller variables; then we
+# need to combine the two dicts when running the test. This seems to be
+# the only way to do it in ansible.
+- name: "{{ test.name }} | define local environment"
+  set_fact:
+    local_environment:
+      TEST_NAME:    "{{ test.name }}"
+      TEST_PACKAGE: "{{ test.package }}"
+      TEST_ENV:     "{{ test.environment }}"
+
+- name: "{{ test.name }} | setup/teardown helper | see if exists"
+  local_action: stat path={{ role_path }}/files/helper.{{ test_name_oneword }}.sh
+  register: helper
+
+- name: "{{ test.name }} | setup/teardown helper | install"
+  copy: src=helper.{{ test_name_oneword }}.sh dest=/tmp/helper.sh
+  when: helper.stat.exists
+
+# This is what runs the BATS tests.
+- name: "{{ test.name }} | run test"
+  script: ./run_bats_tests.sh
+  args:
+    chdir: /usr/share/{{ test.package }}/test/system
+  become: "{{ true if test.become is defined else false }}"
+  become_user: "{{ rootless_user }}"
+  environment: "{{ local_environment | combine(test.environment) }}"
+
+# BATS tests will always exit zero and should leave behind two files:
+# a full log (test.bats.log) and a one-line PASS/FAIL file (.summary.log)
+- name: "{{ test.name }} | pull logs"
+  fetch:
+    src: "/tmp/test.{{ item }}.log"
+    dest: "{{ artifacts }}/test.{{ test_name_oneword }}.{{ item }}.log"
+    flat: yes
+  with_items:
+    - bats
+    - summary
+
+# Collect all the one-line PASS/FAIL results in one file, test.log
+- name: "{{ test.name }} | keep running tally of test results"
+  local_action: shell cat "{{ artifacts }}/test.{{ test_name_oneword }}.summary.log" >>"{{ artifacts }}/test.log"
+
+# ...then delete the oneliner file, to keep things clean.
+- name: "{{ test.name }} | clean up status file"
+  local_action: file dest="{{ artifacts }}/test.{{ test_name_oneword }}.summary.log" state=absent
+
+# FIXME: do the same with results.yml:
+#  - test: {{ test.name }}
+#    result: pass or fail
+#    logs:
+#    - test.{{ test_name_oneword }}.bats.log
+
+- name: "{{ test.name }} | remove remote logs and helpers"
+  file:
+    dest=/tmp/{{ item }}
+    state=absent
+  with_items:
+    - test.bats.log
+    - test.summary.log
+    - helper.sh

tests/roles/set_cgroups/tasks/main.yml

@@ -0,0 +1,61 @@
+---
+# Check the CURRENT cgroup level; we get this from /proc/cmdline
+- name: check current kernel options
+  shell: fgrep systemd.unified_cgroup_hierarchy=0 /proc/cmdline
+  register: result
+  ignore_errors: true
+
+- name: determine current cgroups | assume v2
+  set_fact: current_cgroups=2
+
+- name: determine current cgroups | looks like v1
+  set_fact: current_cgroups=1
+  when: result is succeeded
+
+- debug:
+    msg: "want: v{{ want_cgroups }}  actual: v{{ current_cgroups }}"
+
+# Update grubenv file to reflect the desired cgroup level
+- name: remove cgroup option from kernel flags
+  shell:
+    cmd: sed -i -e "s/^\(kernelopts=.*\)systemd\.unified_cgroup_hierarchy=.\(.*\)/\1\2/" /boot/grub2/grubenv
+    warn: false
+
+- name: add it with the desired value
+  shell:
+    cmd: sed -i -e "s/^\(kernelopts=.*\)/\1 systemd.unified_cgroup_hierarchy=0/" /boot/grub2/grubenv
+    warn: false
+  when: want_cgroups == 1
+
+# If want != have, reboot
+- name: reboot
+  reboot:
+    reboot_timeout: 900
+  register: back_again
+  ignore_errors: yes
+  when: want_cgroups|int != current_cgroups|int
+
+- name: wait and reconnect
+  wait_for_connection:
+    timeout: 900
+  register: back_again2
+  when: want_cgroups|int != current_cgroups|int and not back_again.changed
+
+- set_fact:
+    expected_fstype:
+      - none
+      - tmpfs
+      - cgroup2fs
+
+- name: confirm cgroups setting
+  shell: stat -f -c "%T" /sys/fs/cgroup
+  register: fstype
+
+- debug:
+    msg: "stat(/sys/fs/cgroup) = {{ fstype.stdout }}"
+
+- name: system cgroups is the expected type
+  assert:
+    that:
+      - fstype.stdout == expected_fstype[want_cgroups|int]
+    fail_msg: "stat(/sys/fs/cgroup) = {{ fstype.stdout }} (expected {{ expected_fstype[want_cgroups|int] }})"

tests/test_podman.sh

@@ -1,17 +0,0 @@
-#!/bin/bash -e
-#
-# Simple podman tests
-#
-
-# Log program and kernel versions
-echo "Important package versions:"
-(
-    uname -r
-    rpm -qa | egrep 'podman|conmon|crun|runc|iptable|slirp|systemd|container-selinux' | sort
-) | sed -e 's/^/  /'
-
-# Log environment; or at least the useful bits
-echo "Environment:"
-env | grep -v LS_COLORS= | sort | sed -e 's/^/  /'
-
-bats /usr/share/podman/test/system

tests/test_podman.yml

@@ -1,22 +1,32 @@
 ---
 - hosts: localhost
+  tags:
+    - classic
+    - container
+  vars:
+  - artifacts: ./artifacts
+    rootless_user: testuser
   roles:
-    - role: standard-test-basic
-      tags:
-        - classic
-        - container
-      required_packages:
-        - bats
-        - podman
-        - podman-tests
-      tests:
-        - root-test:
-            dir: ./
-            run: ./test_podman.sh
-            timeout: 15m
-        - rootless-test:
-            # running the test with su doesn't create the directory for fedora user on /run/user/
-            # so create it manually
-            dir: ./
-            run: loginctl enable-linger fedora; su -c ${PWD}/test_podman.sh - fedora
-            timeout: 15m
+  - role: rootless_user_ready
+
+  tasks:
+  # At the start of a run, clean up state. Useful for test reruns.
+  - name: local artifacts directory exists
+    local_action: file path="{{ artifacts }}" state=directory
+
+  - name: remove stale log files
+    local_action: shell rm -f {{ artifacts }}/test*.log
+
+  - name: clear test results
+    local_action: command truncate --size=0 {{ artifacts }}/test.log
+
+  # These are the actual tests: set cgroups vN, then run root/rootless tests.
+  - name: set cgroups and run podman tests
+    include_tasks: test_podman_cgroups_vn.yml
+    loop: [ 2, 1 ]
+    loop_control:
+      loop_var: want_cgroups
+
+  # Postprocessing: check for FAIL or ERROR in any test, exit 1 if so
+  - name: check results
+    include_tasks: check_results.yml

tests/test_podman_cgroups_vn.yml

@@ -0,0 +1,19 @@
+---
+# Requires: 'want_cgroups' variable set to 1 or 2
+- include_role:
+    name: set_cgroups
+- include_role:
+    name: run_bats_tests
+  vars:
+    tests:
+    # Yes, this is horrible duplication, but trying to refactor in ansible
+    # yields even more horrible unreadable code. This is the lesser evil.
+    - name: podman root cgroupsv{{ want_cgroups }}
+      package: podman
+      environment:
+        PODMAN: /usr/bin/podman
+    - name: podman rootless cgroupsv{{ want_cgroups }}
+      package: podman
+      environment:
+        PODMAN: /usr/bin/podman
+      become: true