Add gating tests

Resolves: #1960618

gating.yaml

@@ -0,0 +1,9 @@
+--- !Policy
+product_versions:
+    - rhel-8
+decision_context: osci_compose_gate
+rules:
+    - !PassingTestCaseRule {test_case_name: osci.brew-build.tier0.functional} 
+    # TODO: This is not automated yet and it's not easy to run, so we agreed
+    # with QE we will not run these for 8.1
+    #- !PassingTestCaseRule {test_case_name: manual.sst_platform_storage.lvm2.manual-sts-tier1}

tests/.fmf/version

@@ -0,0 +1 @@
+1

tests/collect_output.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+set -xv
+
+DIR="$1"
+LIST="$2"
+
+[[ -n "$DIR" ]] || {
+  echo "ERROR: Missing DIR" >&2
+  exit 1
+}
+
+cd "$DIR"
+
+# TODO: use find and xargs (?)
+
+passed_logs() {
+  grep '\(passed\|skipped\|warnings\)$' list | sed 's|/|_|' | sed 's| .*|.txt|'
+}
+
+# Remove log file if the test succeeded (we are generating tons of logs)
+echo "Removing output of passed tests..."
+#passed_logs
+
+for f in $(passed_logs); do
+  #echo "  $f"
+  rm -f "$f"
+done
+
+# rm -f $(passed_logs)
+
+if [[ -n "$LIST" ]]; then
+    cat list >> "$LIST"
+fi
+
+# Remove empty files and compress the rest:
+for f in *.txt; do
+  if [[ -s "$f" ]]; then
+    echo "Compressing file '$f'..."
+    ls -l "$f"
+    gzip "$f"
+    ls -l "$f.gz"
+  else
+    echo "Removing empty file '$f'"
+    ls -l "$f"
+    rm -f "$f"
+  fi
+done
+
+for f in *.gz; do
+if [[ -f "$f" ]]; then
+
+  # Use tar so we do not transfer dozens of small files:
+  tar cvf "logfiles.tar" *.gz
+
+  # Clean up:
+  rm -f *.gz
+
+exit 1
+
+fi
+done
+

tests/get_lvm2_tarball.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/sh
+
+set -e
+
+PKG=$1
+
+[[ "$PKG" == lvm2 ]]
+
+CHECK=false
+
+cat sources | \
+  while read SHA512_HEADER TARBALL EQUAL HASH; do
+    if [[ "$SHA512_HEADER" == SHA512 ]]; then
+      TARBALL=$(echo $TARBALL |sed 's/.\(.*\)./\1/')
+      CHECK="sha512sum"
+    else
+      HASH="$SHA512_HEADER"
+      CHECK="md5sum"
+    fi
+    if [[ -f "$TARBALL" ]] && "$CHECK" -c sources; then
+      echo "File already downloaded and verified"
+    else
+      URL="https://sourceware.org/pub/$PKG/releases/$TARBALL"
+      echo "Fetching '$URL'..."
+      curl -k -o "$TARBALL" "$URL"
+      echo "Checking hash of '$TARBALL'..."
+      "$CHECK" -c sources
+    fi
+  done
+

tests/mock_build_and_test.sh

@@ -0,0 +1,85 @@
+#!/bin/bash
+
+set -e
+set -xv
+
+# TODO: Use the role!
+dnf install -y mock
+
+# lsblk and blkid:
+dnf install -y util-linux
+
+# rpmbuild and rpmdev-setuptree
+dnf install -y rpmdevtools rpm-build
+
+# mock:
+dnf install -y mock
+
+# gather info about the system and environment:
+set
+
+pwd
+ls -al
+
+uname -a
+
+rpm -qa | grep -F lvm2 | :
+rpm -qa | grep -F device-mapper | :
+
+which lsblk | :
+lsblk | :
+which blkid | :
+blkid | :
+
+find / -name lvm2.spec
+
+# NOTE: The following does not work - it will download src.rpm from default repo, not the one under test...
+#################################################################################
+## compile the test suite - using rpmbuild and mock
+#################################################################################
+#rpmdev-setuptree
+#
+#dnf download --source lvm2
+#rpm -ivh lvm2*.src.rpm
+#
+#find / -name '*.patch'
+#exit 0
+##rpmbuild -bs
+
+################################################################################
+# compile the test suite - using mock
+################################################################################
+if [[ -f lvm2.spec ]]; then
+
+mock_profile() {
+(
+  ID=
+  VERSION_ID=
+  source /etc/os-release ""
+  [[ -n "$ID" && -n "$VERSION_ID" ]] || exit 1
+  echo "${ID}-${VERSION_ID}-$(arch)"
+)
+}
+
+MOCK_PROFILE="$(mock_profile)"
+
+rm -rf "/var/lib/mock/$MOCK_PROFILE/result/"
+
+#mock -r "$MOCK_PROFILE" --init
+#mock -r "$MOCK_PROFILE" --installdeps lvm2.spec
+# FIXME: /var/str is controlled by tenv_workdir ansible variable - need an env.variable!
+#mock -r "$MOCK_PROFILE" --buildsrpm --no-clean --spec lvm2.spec --sources "/var/str/source" --short-circuit=build
+#ls -l
+
+find / -name 'lvm2*.src.rpm'
+mock -r "$MOCK_PROFILE" --rebuild --no-clean ~/rpmbuild/SRPMS/lvm2*.src.rpm
+
+find / -name 'lvm2-2*.rpm'
+
+[[ -n $(find / -name 'lvm2-testsuite-2*.rpm') ]]
+
+ls -l
+#dnf install -y ~/rpmbuild/RPMS/lvm2-testsuite-*.rpm
+dnf install -y /var/lib/mock/"$MOCK_PROFILE"/result/lvm2-testsuite-2*.rpm
+fi
+

tests/provision.fmf

@@ -0,0 +1,6 @@
+---
+# lvm2 testsuite requires at least 2G of RAM to run smoothly in /dev/shm
+standard-inventory-qcow2:
+    qemu:
+        m: 3G
+

tests/refresh.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+refresh_dirs() {
+  local dir=
+  while [[ -n "$*" ]]; do
+    if [[ -d "$1" ]]; then
+      rm -rf "$1"
+    fi
+    mkdir -p "$(dirname "$1")"
+    mkdir "$1"
+    shift
+  done
+}
+
+refresh_dirs ~/rpmbuild/SRPMS ~/rpmbuild/RPMS
+refresh_dirs /tmp/source

tests/run_tests.yml

@@ -0,0 +1,77 @@
+---
+- name: Echo test header
+  shell: |
+    MSG="Running test case {{ test_case }}"
+    echo "*** $MSG ***" >/dev/console
+    logger -p user.notify "$MSG"
+    true
+
+- name: Create artifacts directory
+  file:
+    state: directory
+    path: "{{ remote_artifacts }}/{{ test_case }}"
+
+- name: Run the testsuite
+  #  lvm2-testsuite --batch --flavours "{{ flavours.stdout_lines|join(',')}}" --only "{{ run_tests|default([])|join(',') }}" --skip "{{ skip_tests|default([])|join(',') }}" 2>&1 | tee "lvm2-testsuite.out"
+  shell: |
+    # Run testsuite in RAM disk:
+    mkdir -p /dev/shm/lvm2-slave
+    export LVM_TEST_DIR=/dev/shm/lvm2-slave
+    mount -o remount,dev /dev/shm
+    # Protection against some of our non-deterministic tests
+    # TODO: Add diff for each test run, so it is easier to tell whether test is broken, or there is an issue
+    RESULT=1
+    ITERATION=0
+    while [[ "$RESULT" -ne 0 && "$ITERATION" -lt {{ iterations|default(3) }} ]]; do
+      let ITERATION++
+      mkdir "$ITERATION"
+      cd "$ITERATION"
+      echo "Running iteration $ITERATION..."
+      lvm2-testsuite --batch --flavours "{{ flavours.stdout_lines|join(',')}}" --only "{{ test_case }}" 2>&1 | tee "lvm2-testsuite.out"
+      RESULT="${PIPESTATUS[0]}"
+      cd ".."
+    done
+    cp -R "$ITERATION"/* "."
+    exit "$RESULT"
+  args:
+    chdir: "{{ remote_artifacts }}/{{ test_case }}"
+  register: lvm2_testsuite_run
+  ignore_errors: yes
+
+- name: Collect artifacts
+  shell: |
+    "{{ workdir }}/collect_output.sh" "{{ remote_artifacts }}/{{ test_case }}" "{{ remote_artifacts }}/list"
+  ignore_errors: yes
+  register: lvm2_testsuite_results
+
+#- name: Find artifacts
+#  find:
+#    path: "{{ remote_artifacts }}"
+#    patterns: "*.txt"
+#  register: artifacts_files
+
+- name: Fetch output
+  fetch:
+    dest: "{{ artifacts }}/{{ test_case }}/"
+    src: "{{ remote_artifacts }}/{{ test_case }}/{{ logfile }}"
+    flat: yes
+  with_items:
+    - lvm2-testsuite.out
+    - journal
+    - list
+    - logfiles.tar
+  loop_control:
+    loop_var: logfile
+  when: lvm2_testsuite_results.failed
+
+- name: Fetch list
+  fetch:
+    dest: "{{ artifacts }}/"
+    src: "{{ remote_artifacts }}/list"
+    flat: yes
+
+- name: Summary
+  shell: |
+    cut -d" " -f2 "{{ remote_artifacts }}/list" | sort | uniq -c
+    grep "failed$" "{{ remote_artifacts }}/list" || :
+

tests/tests.yml

@@ -0,0 +1,338 @@
+- hosts: localhost
+  vars:
+    workdir: /tmp/testwd
+    remote_artifacts: /tmp/testwd/artifacts
+    srcdir: /tmp/source
+    package: lvm2
+    fetch_all: true
+    #fetch_gzip: true
+    #fetch_tar: true
+    install_packages: true
+    #filter_flavours: '| grep -v "lockd\|polld"' # Ignore lockd and polld
+    filter_flavours: '| grep "vanilla"' # Run vanilla only
+    #run_tests:
+    #  - vg
+    ignore_failures:
+      # tests need updating - remove after verified:
+      - lvconvert-thin.sh
+      - lvcreate-cache.sh
+      - lvcreate-thin-big.sh
+      # unstable results - should be warnings only
+      - lvconvert-cache-abort.sh
+      - lvconvert-mirror.sh
+      - dmeventd-restart.sh # TODO: could be an issue!
+      - nomda-restoremissing.sh # Unstable test, investigate
+      - api/dbus # New experimental VDO code not working yet
+      - open-file-limit.sh # Unstable test
+      # these are known issues with md not cleaning up after itself:
+      #- shell/duplicate-pvs-md0.sh
+      #- shell/duplicate-pvs-md1.sh
+      - shell/integrity # An unknown issue, likely in kernel
+      - writecache-blocksize # A new test not working properly
+    skip_tests:
+      - read-ahead.sh # tuned is interfering with tests
+
+      # VDO is not supported yet:
+      - vdo # 8.4.0 kernel kmod-kvdo is broken
+      - lvconvert-cache-vdo.sh # TODO: likely an issue!
+      - lvcreate-vdo-cache.sh # TODO: likely an issue!
+      - lvconvert-cache-abort.sh # TODO: likely an issue!
+
+      # TODO: Check and update list of known issues and unstable tests
+      - shell/000-basic.sh # this fails if there is different version of installed lvm2 and lvm2-testsuite (and we do not want this to fail gating)
+      - shell/fsadm-crypt.sh # broken on RHEL-8 - requires input on stdin
+      - lvconvert-raid-reshape.sh # blocking forever today, worked fine on Monday. kernel issue?
+      # these are known issues with md not cleaning up after itself:
+      #- shell/duplicate-pvs-md0.sh
+      #- shell/duplicate-pvs-md1.sh
+
+      - shell/lvconvert-raid-takeover.sh # did this ever work?
+      - shell/lvm-on-md.sh # check the test!
+      - shell/thin-flags.sh # check the test!
+
+      # these may take considerable amount of time and consume lot of RAM - we are running these on our infrastructure:
+      # vgchange-many does not work in CI - it is timing out writing large amounts of data
+      - shell/vgchange-many.sh
+      #- shell/vgcreate-many-pvs.sh
+      #- shell/thin-many-dmeventd.sh
+
+      # NOTE: Uncomment the following to quickly check the test itself is working
+      #- vg
+      #- lv
+      #- pv
+      #- raid
+      #- mirror
+      #- python
+      #- poll
+      #- thin
+      #- scan
+      #- dbus
+      #
+  tags:
+    - classic
+
+  tasks:
+
+  - name: Set variables
+    set_fact:
+      artifacts: "{{ artifacts|default(lookup('env', 'TEST_ARTIFACTS') or './artifacts') }}"
+
+  - name: str-common-init
+    include_role:
+      name: str-common-init
+
+  - name: Install basic tools
+    dnf:
+      name:
+        - python3-libselinux
+        - sed
+        - tar
+        - util-linux
+        - psmisc # killall is required by testsuite
+        - nmap-ncat # nc required by lvmpolld tests (or socat)
+        - binutils # strings is required by some tests
+        - strace
+        - vim-common # xxd required by some tests
+        - mdadm
+        - vdo
+        - kmod-kvdo
+      state: present
+
+  - name: See RAM and disks
+    shell: |
+      set -xv
+      free
+      mount
+      df
+      lsblk
+      true
+
+  - name: Install rsync on localhost
+    dnf:
+      name: rsync
+      state: present
+    delegate_to: 127.0.0.1
+    ignore_errors: yes
+    register: has_rsync
+
+  - name: Install rsync on remote
+    dnf:
+      name: rsync
+      state: present
+
+  - name: Create workdir
+    file:
+      state: directory
+      path: "{{ workdir }}"
+      mode: 0755
+
+  - name: Copy scripts to remote machine
+    copy:
+      src: "{{ playbook_dir }}/{{ item }}"
+      dest: "{{ workdir }}/"
+      mode: 0755
+    with_items:
+      - get_lvm2_tarball.sh
+      - refresh.sh
+      - collect_output.sh
+      #- mock_build_and_test.sh
+
+  - name: Check lvm2-testsuite is present
+    dnf:
+      name: lvm2-testsuite
+      state: present
+    ignore_errors: yes
+    register: lvm2_testsuite
+
+  - name: Build and install lvm2-testsuite if not available
+    when: lvm2_testsuite.failed
+    block:
+
+      - name: Make sure dependencies for retrieving the sources are installed
+        dnf:
+          name:
+            - curl
+            - wget
+            - rpm-build
+            - xz
+            #- mock
+            - rpmdevtools
+            - make
+
+      - name: Copy spec file to remote machine
+        copy:
+          src: "{{ playbook_dir }}/../{{package}}.spec"
+          dest: "{{ workdir }}/"
+
+#      - name: Display path to patches
+#        debug:
+#          msg: "{{ playbook_dir + '/../*.patch' }}"
+#
+#      - name: Display patches
+#        debug:
+#          msg: "{{ lookup('fileglob', playbook_dir + '/../*.patch') }}"
+
+      - name: Copy patches to remote machine
+        copy:
+          src: "{{ item }}"
+          dest: "{{ workdir }}/"
+        with_fileglob: "{{ playbook_dir }}/../*.patch"
+
+      - name: Copy sources file to remote machine
+        copy:
+          src: "{{ playbook_dir }}/../sources"
+          dest: "{{ workdir }}/"
+
+      - name: Enable testsuite in the spec file
+        shell: |
+          sed -i 's/enable_testsuite .*/enable_testsuite 1/' lvm2.spec
+        args:
+          chdir: "{{ workdir }}"
+
+      # This would be handled by mock if available. And is not needed at all on Fedora
+      # TODO: Use conditional for the Fedora part!
+      - name: Enable buildroot repo
+        shell: dnf -y config-manager --set-enabled rhel-buildroot
+        ignore_errors: yes
+
+      - name: Install build deps
+        shell: dnf -y build-dep {{ workdir }}/{{ package }}.spec
+
+      - name: Run the refresh script
+        shell: |
+          ./refresh.sh
+        args:
+          chdir: "{{ workdir }}"
+
+      - name: Run the source-retrieval script
+        shell: |
+          ./get_lvm2_tarball.sh {{ package }}
+        args:
+          chdir: "{{ workdir }}"
+
+      - name: Build RPMs
+        shell: |
+          #rpmbuild -bp lvm2.spec --define "_sourcedir $PWD" --define "_specdir $PWD" --define "_builddir {{ srcdir }}"
+          #rpmbuild -bs lvm2.spec --define "_sourcedir $PWD" --define "_specdir $PWD" --define "_builddir {{ srcdir }}"
+          rpmbuild -ba lvm2.spec --define "_sourcedir $PWD" --define "_specdir $PWD" --define "_builddir {{ srcdir }}"
+        args:
+          chdir: "{{ workdir }}"
+
+      - name: Install lvm2-testsuite
+        shell: |
+          dnf install -y /root/rpmbuild/RPMS/$(uname -m)/lvm2-testsuite-2.*.rpm
+
+        args:
+          chdir: '{{ workdir }}'
+
+#      - name: Build and install lvm2-testsuite
+#        shell: |
+#          ./mock_build_and_test.sh
+#        args:
+#          chdir: "{{ workdir }}"
+
+  - name: Install lvm2 packages if requested
+    dnf:
+      state: present
+      name:
+        - lvm2
+        - lvm2-lockd
+        - lvm2-dbusd
+        - device-mapper-event
+    when: install_packages|default(false)
+
+  - name: Disable services interfering with tests
+    systemd:
+      name: "{{item}}"
+      masked: yes
+      enabled: no
+      state: stopped
+    ignore_errors: yes
+    with_items:
+      - lvm2-lvmpolld.service
+      - lvm2-lvmpolld.socket
+      - lvm2-monitor.service
+      - dm-event.socket
+      - dm-event.service
+      - lvm2-pvscan@.service
+
+  # The above systemd module does not like templates...
+  - name: Disable pvscan service
+    shell: |
+        systemctl mask lvm2-pvscan@.service
+    ignore_errors: yes
+
+  - name: Disable monitoring
+    shell: |
+      sed -i 's/monitoring *= *.*/monitoring = 0/' /etc/lvm/lvm.conf
+
+  - name: Check environment
+    shell: |
+      lvmconfig --typeconfig diff
+      systemctl -a
+
+  - name: Create artifacts directory
+    file:
+      state: directory
+      path: "{{ remote_artifacts }}"
+
+  - name: Gather flavours
+    shell: |
+          ls -1 flavour-udev-* | grep -v "lvmlockd-dlm\|lvmlockd-sanlock" {{ filter_flavours|default('') }} | sed 's/^flavour-//'
+    args:
+      chdir: /usr/share/lvm2-testsuite/lib
+    register: flavours
+
+#  - name: Find flavours
+#    find:
+#      contains: '^flavour-'
+#      paths: /usr/share/lvm2-testsuite/lib
+#    register: flavours
+#    # TODO: should use flavours.files when using find module, must drop flavour- prefix
+
+#  - name: Debug flavours
+#    debug:
+#      var: flavours
+#
+#  - name: Debug skip
+#    debug:
+#      msg: |
+#        '\({{ skip_tests|join("\|") }}\)'
+
+  - name: Gather tests
+    shell: |
+      find . -name "*.sh" | sed 's/^\.\///' | grep -v '\({{ skip_tests|default(["qwqwasdf"])|join("\|") }}\)' | grep '\({{ run_tests|default(["."])|join("\|") }}\)' | sort
+    args:
+      chdir: /usr/share/lvm2-testsuite
+    register: lvm2_tests
+
+#  - name: Debug lvm2_tests
+#    debug:
+#      var: lvm2_tests
+
+  - name: Run tests
+    include_tasks: run_tests.yml
+    with_items: "{{ lvm2_tests.stdout_lines }}"
+    loop_control:
+      loop_var: test_case
+
+  # NOTE: test.log is required by CI - shall we append to it?
+  - name: Fetch list with results as test.log
+    fetch:
+      dest: "{{ artifacts }}/test.log"
+      src: "{{ remote_artifacts }}/list"
+      flat: yes
+
+#  - name: Check artifacts size on remote
+#    shell: |
+#      du -s "{{ remote_artifacts }}/"
+#
+#  - name: Check artifacts size on local
+#    shell: |
+#      du -s "{{ artifacts }}/"
+#    delegate_to: 127.0.0.1
+
+  - name: Report failures
+    shell: |
+      ! grep -v '\(passed\|skipped\|warnings\)' "{{ remote_artifacts }}/list" | grep -v '\({{ ignore_failures|default(["qwqwasdf"])|join("\|") }}\)'
+