From f7ecf95dc70628d1e55014b956d6c1279ff946b7 Mon Sep 17 00:00:00 2001
From: Milos Prchlik <mprchlik@redhat.com>
Date: Mon, 18 Oct 2021 16:01:51 +0200
Subject: [PATCH] tests: switch to TMT

---
 .fmf/version                                  |  1 +
 tests/clang-format-diff/main.fmf              |  4 ++
 .../clang-format-diff/{runtest.sh => test.sh} |  4 +-
 tests/default.fmf                             | 19 ++++++
 tests/fedora-flags/main.fmf                   |  5 ++
 tests/fedora-flags/{runtest.sh => test.sh}    |  4 +-
 tests/integration-test-suite/runtest.sh       | 38 -----------
 tests/libomp/main.fmf                         | 13 ++++
 tests/libomp/{runtest.sh => test.sh}          |  0
 tests/llvm-test-suite/main.fmf                |  6 ++
 tests/llvm-test-suite/test.sh                 |  7 +++
 tests/llvm-toolchain/main.fmf                 | 12 ++++
 tests/llvm-toolchain/{runtest.sh => test.sh}  |  4 +-
 tests/pocl/main.fmf                           | 16 +++++
 tests/pocl/test.sh                            |  7 +++
 tests/rhbz_1647130/main.fmf                   |  4 ++
 tests/rhbz_1647130/{runtest.sh => test.sh}    |  4 +-
 tests/rhbz_1657544/main.fmf                   |  4 ++
 tests/rhbz_1657544/{runtest.sh => test.sh}    |  4 +-
 tests/rhbz_482491/main.fmf                    |  5 ++
 tests/rhbz_482491/test.sh                     |  3 +
 tests/tests-libomp.fmf                        | 16 +++++
 tests/tests-libomp.yml                        | 21 -------
 tests/tests-pocl.fmf                          | 18 ++++++
 tests/tests.yml                               | 63 -------------------
 tests/testspocl.yml                           | 28 ---------
 tests/toolchains/main.fmf                     |  7 +++
 tests/toolchains/{runtest.sh => test.sh}      |  2 +-
 28 files changed, 155 insertions(+), 164 deletions(-)
 create mode 100644 .fmf/version
 create mode 100644 tests/clang-format-diff/main.fmf
 rename tests/clang-format-diff/{runtest.sh => test.sh} (79%)
 create mode 100644 tests/default.fmf
 create mode 100644 tests/fedora-flags/main.fmf
 rename tests/fedora-flags/{runtest.sh => test.sh} (94%)
 delete mode 100755 tests/integration-test-suite/runtest.sh
 create mode 100644 tests/libomp/main.fmf
 rename tests/libomp/{runtest.sh => test.sh} (100%)
 create mode 100644 tests/llvm-test-suite/main.fmf
 create mode 100644 tests/llvm-test-suite/test.sh
 create mode 100644 tests/llvm-toolchain/main.fmf
 rename tests/llvm-toolchain/{runtest.sh => test.sh} (95%)
 create mode 100644 tests/pocl/main.fmf
 create mode 100755 tests/pocl/test.sh
 create mode 100644 tests/rhbz_1647130/main.fmf
 rename tests/rhbz_1647130/{runtest.sh => test.sh} (91%)
 create mode 100644 tests/rhbz_1657544/main.fmf
 rename tests/rhbz_1657544/{runtest.sh => test.sh} (66%)
 create mode 100644 tests/rhbz_482491/main.fmf
 create mode 100755 tests/rhbz_482491/test.sh
 create mode 100644 tests/tests-libomp.fmf
 delete mode 100644 tests/tests-libomp.yml
 create mode 100644 tests/tests-pocl.fmf
 delete mode 100644 tests/tests.yml
 delete mode 100644 tests/testspocl.yml
 create mode 100644 tests/toolchains/main.fmf
 rename tests/toolchains/{runtest.sh => test.sh} (99%)

diff --git a/.fmf/version b/.fmf/version
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/.fmf/version
@@ -0,0 +1 @@
+1
diff --git a/tests/clang-format-diff/main.fmf b/tests/clang-format-diff/main.fmf
new file mode 100644
index 0000000..8eadb00
--- /dev/null
+++ b/tests/clang-format-diff/main.fmf
@@ -0,0 +1,4 @@
+summary: clang-format-diff
+test: ./test.sh
+require:
+  - clang-tools-extra
diff --git a/tests/clang-format-diff/runtest.sh b/tests/clang-format-diff/test.sh
similarity index 79%
rename from tests/clang-format-diff/runtest.sh
rename to tests/clang-format-diff/test.sh
index d27ee13..d6fcb47 100755
--- a/tests/clang-format-diff/runtest.sh
+++ b/tests/clang-format-diff/test.sh
@@ -1,6 +1,4 @@
-#!/bin/bash
-
-set -ex
+#!/bin/sh -eux
 
 # Check that clang-format-diff is in PATH.
 # rhbz#1939018
diff --git a/tests/default.fmf b/tests/default.fmf
new file mode 100644
index 0000000..50db994
--- /dev/null
+++ b/tests/default.fmf
@@ -0,0 +1,19 @@
+summary: Clang tests
+discover:
+    - name: Local tests
+      how: fmf
+      filter: "tag:-not-in-default"
+    - name: "LLVM's integration test suite"
+      how: fmf
+      url: https://src.fedoraproject.org/rpms/llvm.git
+      ref: rawhide
+      test: integration-test-suite
+execute:
+    how: tmt
+prepare:
+    - name: packages
+      how: install
+      package: clang
+provision:
+  hardware:
+    memory: ">= 4 GiB"
diff --git a/tests/fedora-flags/main.fmf b/tests/fedora-flags/main.fmf
new file mode 100644
index 0000000..118336e
--- /dev/null
+++ b/tests/fedora-flags/main.fmf
@@ -0,0 +1,5 @@
+summary: fedora-flags
+test: ./test.sh
+require:
+  - annobin
+  - redhat-rpm-config
diff --git a/tests/fedora-flags/runtest.sh b/tests/fedora-flags/test.sh
similarity index 94%
rename from tests/fedora-flags/runtest.sh
rename to tests/fedora-flags/test.sh
index 96d1cc8..75f674f 100755
--- a/tests/fedora-flags/runtest.sh
+++ b/tests/fedora-flags/test.sh
@@ -1,6 +1,6 @@
-#!/bin/bash
+#!/bin/sh -eux
 
-set -ex pipefail
+set pipefail
 
 cflags=`rpm -D '%toolchain clang' -E %{build_cflags}`
 cxxflags=`rpm -D '%toolchain clang' -E %{build_cxxflags}`
diff --git a/tests/integration-test-suite/runtest.sh b/tests/integration-test-suite/runtest.sh
deleted file mode 100755
index 1af4e4a..0000000
--- a/tests/integration-test-suite/runtest.sh
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/bin/bash
-
-usage() {
-    echo "usage `basename $0` [OPTIONS]"
-    echo "  --threads NUM       The number of threads to use for running tests."
-}
-
-thread_args=""
-repo_url=https://github.com/opencollab/llvm-toolchain-integration-test-suite
-
-while [ $# -gt 0 ]; do
-    case $1 in
-        --threads)
-            shift
-            threads="$1"
-            ;;
-        * )
-            echo "unknown option: $1"
-            echo ""
-            usage
-            exit 1
-            ;;
-    esac
-    shift
-done
-
-if [ -n "$threads" ]; then
-  thread_args="-j$threads"
-fi
-
-set -xe
-
-cd $(mktemp -d -p /var/tmp)
-git clone $repo_url
-cd llvm-toolchain-integration-test-suite
-mkdir _build && cd _build
-cmake .. -GNinja
-ninja $thread_args check
diff --git a/tests/libomp/main.fmf b/tests/libomp/main.fmf
new file mode 100644
index 0000000..a2c9403
--- /dev/null
+++ b/tests/libomp/main.fmf
@@ -0,0 +1,13 @@
+# TODO: get fitting summary - what's the purpose of this test? Comment from the STI version says:
+#
+# We want to make sure libomp is not already present on the system to ensure
+# that clang pulls in the correct libomp dependencies when it is installed.
+summary: libomp tests
+# This test requires a special preparation, see `/tests/tests-libomp` plan. As such,
+# it shouldn't be part of the `/tests/default` plan.
+tag:
+  - not-in-default
+test: ./test.sh
+require:
+  - clang
+  - libomp
diff --git a/tests/libomp/runtest.sh b/tests/libomp/test.sh
similarity index 100%
rename from tests/libomp/runtest.sh
rename to tests/libomp/test.sh
diff --git a/tests/llvm-test-suite/main.fmf b/tests/llvm-test-suite/main.fmf
new file mode 100644
index 0000000..4bdf182
--- /dev/null
+++ b/tests/llvm-test-suite/main.fmf
@@ -0,0 +1,6 @@
+# TODO: once llvm-test-suite is converted to TMT, we can just link the test from plan.
+summary: Run tests from llvm-test-suite package
+test: ./test.sh
+require:
+  - git
+  - clang
diff --git a/tests/llvm-test-suite/test.sh b/tests/llvm-test-suite/test.sh
new file mode 100644
index 0000000..85418bc
--- /dev/null
+++ b/tests/llvm-test-suite/test.sh
@@ -0,0 +1,7 @@
+#!/bin/sh -eux
+
+# TODO: tmt does not support a remote git repo as a requirement, one has to clone it "manually".
+
+git clone --depth 1 https://src.fedoraproject.org/rpms/llvm-test-suite.git llvm-test-suite
+cd llvm-test-suite/tests/test-suite
+./runtest.sh
diff --git a/tests/llvm-toolchain/main.fmf b/tests/llvm-toolchain/main.fmf
new file mode 100644
index 0000000..ff88230
--- /dev/null
+++ b/tests/llvm-toolchain/main.fmf
@@ -0,0 +1,12 @@
+# TODO REVIEW: better summary
+summary: ""
+test: ./test.sh
+require:
+  - clang
+  - lld
+  - compiler-rt
+  - libcxx-devel
+  - libcxx-static
+  - glibc-devel
+  - glibc-static
+  - gcc
diff --git a/tests/llvm-toolchain/runtest.sh b/tests/llvm-toolchain/test.sh
similarity index 95%
rename from tests/llvm-toolchain/runtest.sh
rename to tests/llvm-toolchain/test.sh
index eeebda5..4dde26c 100755
--- a/tests/llvm-toolchain/runtest.sh
+++ b/tests/llvm-toolchain/test.sh
@@ -1,8 +1,8 @@
-#!/bin/sh
+#!/bin/sh -eux
 
 # Tests for using a full LLVM toolchain: clang + compiler-rt + libcxx + lld
 
-set -ex pipefail
+set pipefail
 
 # Test compile a C program.
 cat << EOF | \
diff --git a/tests/pocl/main.fmf b/tests/pocl/main.fmf
new file mode 100644
index 0000000..92c244a
--- /dev/null
+++ b/tests/pocl/main.fmf
@@ -0,0 +1,16 @@
+# TODO REVIEW: get fitting summary - what's the purpose of this test? Comment from the STI version says:
+#
+# This is separate from tests.yml because we want to run the tests without
+# clang installed.
+summary: pocl
+# This test requires a special preparation, see `/tests/tests-pocl` plan. As such,
+# it shouldn't be part of the `/tests/default` plan.
+tag:
+  - not-in-default
+# rhbz#1582884
+test: ./test.sh
+require:
+  - git
+  - ocl-icd-devel
+  - pocl
+  - gcc
diff --git a/tests/pocl/test.sh b/tests/pocl/test.sh
new file mode 100755
index 0000000..fd9f92b
--- /dev/null
+++ b/tests/pocl/test.sh
@@ -0,0 +1,7 @@
+#!/bin/sh -eux
+
+# TODO: tmt does not support a remote git repo as a requirement, one has to clone it "manually".
+
+git clone --depth 1 https://src.fedoraproject.org/rpms/pocl.git pocl
+cd pocl/tests/simple-opencl-no-clang
+./runtest.sh
diff --git a/tests/rhbz_1647130/main.fmf b/tests/rhbz_1647130/main.fmf
new file mode 100644
index 0000000..40c4aa1
--- /dev/null
+++ b/tests/rhbz_1647130/main.fmf
@@ -0,0 +1,4 @@
+summary: RHBZ1647130
+test: ./test.sh
+require:
+  - clang
diff --git a/tests/rhbz_1647130/runtest.sh b/tests/rhbz_1647130/test.sh
similarity index 91%
rename from tests/rhbz_1647130/runtest.sh
rename to tests/rhbz_1647130/test.sh
index ab515c7..7fb47ba 100755
--- a/tests/rhbz_1647130/runtest.sh
+++ b/tests/rhbz_1647130/test.sh
@@ -1,6 +1,4 @@
-#!/bin/sh
-set -e
-set -x
+#!/bin/sh -eux
 
 tmp_cpp=`mktemp -t XXXXX.cpp`
 tmp_dir=`mktemp -d`
diff --git a/tests/rhbz_1657544/main.fmf b/tests/rhbz_1657544/main.fmf
new file mode 100644
index 0000000..223c5f5
--- /dev/null
+++ b/tests/rhbz_1657544/main.fmf
@@ -0,0 +1,4 @@
+summary: RHBZ1657544
+test: ./test.sh
+require:
+  - clang
diff --git a/tests/rhbz_1657544/runtest.sh b/tests/rhbz_1657544/test.sh
similarity index 66%
rename from tests/rhbz_1657544/runtest.sh
rename to tests/rhbz_1657544/test.sh
index 758de0b..58c088f 100755
--- a/tests/rhbz_1657544/runtest.sh
+++ b/tests/rhbz_1657544/test.sh
@@ -1,6 +1,4 @@
-#!/bin/sh
-set -e
-set -x
+#!/bin/sh -eux
 
 clang++ from_chars.cpp
 ./a.out 100 | grep 100
diff --git a/tests/rhbz_482491/main.fmf b/tests/rhbz_482491/main.fmf
new file mode 100644
index 0000000..cac70ae
--- /dev/null
+++ b/tests/rhbz_482491/main.fmf
@@ -0,0 +1,5 @@
+summary: rhbz-482491
+test: ./test.sh
+require:
+  - clang
+  - libgcc
diff --git a/tests/rhbz_482491/test.sh b/tests/rhbz_482491/test.sh
new file mode 100755
index 0000000..2360238
--- /dev/null
+++ b/tests/rhbz_482491/test.sh
@@ -0,0 +1,3 @@
+#!/bin/sh -eux
+
+find /usr -name 'libgcc_s.so*' && echo "int main(){}" | clang -v -x c -
diff --git a/tests/tests-libomp.fmf b/tests/tests-libomp.fmf
new file mode 100644
index 0000000..c80eb11
--- /dev/null
+++ b/tests/tests-libomp.fmf
@@ -0,0 +1,16 @@
+summary: libomp tests
+discover:
+    how: fmf
+    test: libomp
+execute:
+    how: tmt
+prepare:
+    # We want to make sure libomp is not already present on the system to ensure
+    # that clang pulls in the correct libomp dependencies when it is installed.
+    - name: Drop libomp
+      how: shell
+      script: |
+        dnf erase -y libomp libomp-devel clang clang-libs
+provision:
+  hardware:
+    memory: ">= 4 GiB"
diff --git a/tests/tests-libomp.yml b/tests/tests-libomp.yml
deleted file mode 100644
index a9db418..0000000
--- a/tests/tests-libomp.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-- hosts: localhost
-  pre_tasks:
-    # We want to make sure libomp is not already present on the system to ensure
-    # that clang pulls in the correct libomp dependencies when it is installed.
-    - name: Uninstall libomp
-      package:
-        name: "{{ item }}"
-        state: absent
-      with_items:
-        - libomp
-        - libomp-devel
-        - clang
-        - clang-libs
-  roles:
-  - role: standard-test-basic
-    tags:
-      - classic
-    required_packages:
-      - clang
-    tests:
-      - libomp
diff --git a/tests/tests-pocl.fmf b/tests/tests-pocl.fmf
new file mode 100644
index 0000000..86c91a9
--- /dev/null
+++ b/tests/tests-pocl.fmf
@@ -0,0 +1,18 @@
+summary: pocl tests
+discover:
+    how: fmf
+    test: pocl
+execute:
+    how: tmt
+prepare:
+    # Programs linked against pocl should be able to run without clang installed,
+    # so when we run the test we want to make sure clang is not installed to
+    # verify we haven't introduced an implicit dependency on clang.
+    - name: Drop clang
+      how: shell
+      script: |
+        dnf erase -y clang
+
+provision:
+  hardware:
+    memory: ">= 4 GiB"
diff --git a/tests/tests.yml b/tests/tests.yml
deleted file mode 100644
index 51182b0..0000000
--- a/tests/tests.yml
+++ /dev/null
@@ -1,63 +0,0 @@
-- hosts: localhost
-  roles:
-  - role: standard-test-basic
-    tags:
-      - classic
-    required_packages:
-      - clang
-      - llvm-test-suite
-      - ninja-build
-      - gcc-x86_64-linux-gnu
-      # FIXME: It would be nice if we could only install dependencies for a
-      # single test rather than installing dependencies for all tests.  This
-      # will help us catch bugs with implicit package dependencies e.g.
-      # package A depends on package B but does not have an explicit Requires.
-      # These are required for the llvm-toolchain test:
-      - clang
-      - lld
-      - compiler-rt
-      - libcxx-devel
-      - libcxx-static
-      - glibc-devel
-      - glibc-static
-      - gcc
-      # Required for fedora-flags:
-      - annobin
-      - redhat-rpm-config
-      # Required for clang-format-diff
-      - clang-tools-extra
-      # the requirements below are for the integration suite
-      - cmake
-      - llvm-devel
-      - clang
-      - clang-analyzer
-      - clang-tools-extra
-      - compiler-rt
-      - ninja-build
-      - libcxx-devel
-      - libomp-devel
-      - python-lit
-      - lld
-      - lldb
-      - git
-      - make
-      - libstdc++-static
-      - clang-devel
-
-    repositories:
-    - repo: "https://src.fedoraproject.org/rpms/llvm-test-suite.git"
-      dest: "llvm-test-suite"
-    tests:
-      - rhbz#482491:
-          dir: ./
-          run: find /usr -name 'libgcc_s.so*' && echo "int main(){}" | clang -v -x c -
-      - llvm-test-suite/tests/test-suite
-      # ABI test suite is too greedy on the FS
-      #- llvm-test-suite/abi-test-suite
-      - rhbz_1657544
-      - rhbz_1647130
-      - llvm-toolchain
-      - fedora-flags
-      - toolchains
-      - clang-format-diff
-      - integration-test-suite
diff --git a/tests/testspocl.yml b/tests/testspocl.yml
deleted file mode 100644
index 7189eac..0000000
--- a/tests/testspocl.yml
+++ /dev/null
@@ -1,28 +0,0 @@
-# This is separate from tests.yml because we want to run the tests without
-# clang installed.
-# FIXME: Is there some way we can put this yaml definition in the tests/pocl
-# repo?  Right now we need to copy and paste this everywhere the test is run.
-- hosts: localhost
-  pre_tasks:
-  # Programs linked against pocl should be able to run without clang installed,
-  # so when we run the test we want to make sure clang is not installed to
-  # verify we haven't introduced an implicit dependency on clang.
-  - name: Uninstall clang
-    package:
-      name: clang
-      state: absent
-    tags: classic
-  roles:
-  - role: standard-test-basic
-    tags:
-      - classic
-    repositories:
-    - repo: "https://src.fedoraproject.org/rpms/pocl.git"
-      dest: "pocl"
-    required_packages:
-      - ocl-icd-devel
-      - pocl
-      - gcc
-    tests:
-    # rhbz#1582884
-    - pocl/tests/simple-opencl-no-clang
diff --git a/tests/toolchains/main.fmf b/tests/toolchains/main.fmf
new file mode 100644
index 0000000..86705f4
--- /dev/null
+++ b/tests/toolchains/main.fmf
@@ -0,0 +1,7 @@
+# TODO REVIEW: better summary
+summary: ""
+test: ./test.sh
+# TODO REVIEW: are these all requirements? test.sh seems to run quite a lot of stuff, looks like we
+# need more packages from LLVM family.
+require:
+  - clang
diff --git a/tests/toolchains/runtest.sh b/tests/toolchains/test.sh
similarity index 99%
rename from tests/toolchains/runtest.sh
rename to tests/toolchains/test.sh
index c928ae4..f6d77fb 100755
--- a/tests/toolchains/runtest.sh
+++ b/tests/toolchains/test.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -eux
 
 set pipefail