Update to 31

This commit also enables at least some meaningful tests in gating.

Resolves: rhbz#1972427

.gitignore

@@ -20,3 +20,4 @@
 /osbuild-composer-28.tar.gz
 /osbuild-composer-29.tar.gz
 /osbuild-composer-30.tar.gz
+/osbuild-composer-31.tar.gz

0001-PR-1645-test-image-fix-pipeline-exports-for-v2-manifests.patch

@@ -0,0 +1,75 @@
+From 85cc7687415a96db017acaf763d53abbc47d993f Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Ond=C5=99ej=20Budai?= <ondrej@budai.cz>
+Date: Mon, 16 Aug 2021 12:56:55 +0200
+Subject: [PATCH] test/image: fix pipeline exports for v2 manifests
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+Previously, we just assumed that all test manifests are of version 1, or we
+should export the pipeline named assembler. However, this is no longer true
+in RHEL 8.5 and 9 - they are only manifest v2 and they don't have a pipeline
+named assembler.
+
+This commit introduces a new way to guess the export name - if the manifest
+is of version 1, we just export the assembler. In the case v2 manifest, the
+last pipeline is exported.
+
+Signed-off-by: Ondřej Budai <ondrej@budai.cz>
+---
+ cmd/osbuild-image-tests/main_test.go | 32 +++++++++++++++++++++++++---
+ 1 file changed, 29 insertions(+), 3 deletions(-)
+
+diff --git a/cmd/osbuild-image-tests/main_test.go b/cmd/osbuild-image-tests/main_test.go
+index f6cbccc9..f73bab3d 100644
+--- a/cmd/osbuild-image-tests/main_test.go
++++ b/cmd/osbuild-image-tests/main_test.go
+@@ -482,6 +482,34 @@ func testImage(t *testing.T, testcase testcaseStruct, imagePath string) {
+ 	}
+ }
+ 
++// guessPipelineToExport return a best-effort guess about which
++// pipeline should be exported when running osbuild for the testcase
++//
++// If this function detects that this is a version 1 manifest, it
++// always returns "assembler"
++//
++// For manifests version 2, the name of the last pipeline is returned.
++func guessPipelineToExport(rawManifest json.RawMessage) string {
++	const v1ManifestExportName = "assembler"
++	var v2Manifest struct {
++		Version   string `json:"version"`
++		Pipelines []struct {
++			Name string `json:"name,omitempty"`
++		} `json:"pipelines"`
++	}
++	err := json.Unmarshal(rawManifest, &v2Manifest)
++	if err != nil {
++		// if we cannot unmarshal, let's just assume that it's a version 1 manifest
++		return v1ManifestExportName
++	}
++
++	if v2Manifest.Version == "2" {
++		return v2Manifest.Pipelines[len(v2Manifest.Pipelines)-1].Name
++	}
++
++	return v1ManifestExportName
++}
++
+ // runTestcase builds the pipeline specified in the testcase and then it
+ // tests the result
+ func runTestcase(t *testing.T, testcase testcaseStruct, store string) {
+@@ -494,9 +522,7 @@ func runTestcase(t *testing.T, testcase testcaseStruct, store string) {
+ 		require.NoError(t, err, "error removing temporary output directory")
+ 	}()
+ 
+-	// NOTE(akoutsou) 1to2t: new v2 manifests name their last pipeline
+-	// "assembler" for compatibility with v1
+-	exports := []string{"assembler"}
++	exports := []string{guessPipelineToExport(testcase.Manifest)}
+ 	err = runOsbuild(testcase.Manifest, store, outputDirectory, exports)
+ 	require.NoError(t, err)
+ 
+-- 
+2.31.1
+

osbuild-composer.spec

@@ -4,7 +4,7 @@
 
 %global goipath         github.com/osbuild/osbuild-composer
 
-Version:        30
+Version:        31
 
 %gometa
 
@@ -17,7 +17,7 @@ It is compatible with composer-cli and cockpit-composer clients.
 }
 
 Name:           osbuild-composer
-Release:        2%{?dist}
+Release:        1%{?dist}
 Summary:        An image building service based on osbuild
 
 # osbuild-composer doesn't have support for building i686 images
@@ -28,7 +28,8 @@ ExcludeArch:    i686
 License:        ASL 2.0
 URL:            %{gourl}
 Source0:        %{gosource}
-
+Patch0:         0001-PR-1645-test-image-fix-pipeline-exports-for-v2-manifests.patch
+Patch1:         0002-PR-1638-fix-ami.patch
 
 BuildRequires:  %{?go_compiler:compiler(go-compiler)}%{!?go_compiler:golang}
 BuildRequires:  systemd
@@ -47,6 +48,7 @@ BuildRequires:  golang(github.com/coreos/go-systemd/activation)
 BuildRequires:  golang(github.com/deepmap/oapi-codegen/pkg/codegen)
 BuildRequires:  golang(github.com/go-chi/chi)
 BuildRequires:  golang(github.com/google/uuid)
+BuildRequires:  golang(github.com/jackc/pgx/v4)
 BuildRequires:  golang(github.com/julienschmidt/httprouter)
 BuildRequires:  golang(github.com/getkin/kin-openapi/openapi3)
 BuildRequires:  golang(github.com/kolo/xmlrpc)
@@ -54,10 +56,12 @@ BuildRequires:  golang(github.com/labstack/echo/v4)
 BuildRequires:  golang(github.com/gobwas/glob)
 BuildRequires:  golang(github.com/google/go-cmp/cmp)
 BuildRequires:  golang(github.com/gophercloud/gophercloud)
+BuildRequires:  golang(github.com/prometheus/client_golang/prometheus/promhttp)
 BuildRequires:  golang(github.com/stretchr/testify/assert)
 BuildRequires:  golang(github.com/ubccr/kerby)
 BuildRequires:  golang(github.com/vmware/govmomi)
 BuildRequires:  golang(cloud.google.com/go)
+BuildRequires:  golang(gopkg.in/ini.v1)
 %endif
 
 Requires: %{name}-core = %{version}-%{release}
@@ -94,22 +98,6 @@ Obsoletes: osbuild-composer-koji <= 23
 %goprep
 %endif
 
-%if 0%{?fedora} && 0%{?fedora} <= 32
-# Fedora 32 and older ships different kolo/xmlrpc and azure/azblob APIs. We
-# cannot specify build tags in gobuild macro because the macro itself
-# specifies build tags and -tags argument cannot be used more than once.
-# Therefore, this ugly hack with build tags switcharoo is required.
-# Remove when F32 is EOL.
-
-# Remove the build constraint from the wrappers of the old APIs
-sed -i "s$// +build kolo_xmlrpc_oldapi$// +build !kolo_xmlrpc_oldapi$" internal/upload/koji/xmlrpc-response-oldapi.go
-sed -i "s$// +build azblob_oldapi$// +build !azblob_oldapi$" internal/upload/azure/page_blob_url_oldapi.go
-
-# Add a build constraint to the wrappers of the new APIs
-sed -i "s$// +build !kolo_xmlrpc_oldapi$// +build kolo_xmlrpc_oldapi$" internal/upload/koji/xmlrpc-response.go
-sed -i "s$// +build !azblob_oldapi$// +build azblob_oldapi$" internal/upload/azure/page_blob_url.go
-%endif
-
 %if 0%{?fedora} >= 34
 # Fedora 34 and newer ships a newer version of github.com/getkin/kin-openapi
 # package which has a different API than the older ones. Let's make the auto-
@@ -157,6 +145,7 @@ go test -c -tags=integration -ldflags="${TEST_LDFLAGS}" -o _bin/osbuild-weldr-te
 go test -c -tags=integration -ldflags="${TEST_LDFLAGS}" -o _bin/osbuild-image-tests %{goipath}/cmd/osbuild-image-tests
 go test -c -tags=integration -ldflags="${TEST_LDFLAGS}" -o _bin/osbuild-auth-tests %{goipath}/cmd/osbuild-auth-tests
 go test -c -tags=integration -ldflags="${TEST_LDFLAGS}" -o _bin/osbuild-koji-tests %{goipath}/cmd/osbuild-koji-tests
+go test -c -tags=integration -ldflags="${TEST_LDFLAGS}" -o _bin/osbuild-composer-dbjobqueue-tests %{goipath}/cmd/osbuild-composer-dbjobqueue-tests
 go build -tags=integration -ldflags="${TEST_LDFLAGS}" -o _bin/cloud-cleaner %{goipath}/cmd/cloud-cleaner
 
 %endif
@@ -190,10 +179,12 @@ install -m 0755 -vp _bin/osbuild-dnf-json-tests                 %{buildroot}%{_l
 install -m 0755 -vp _bin/osbuild-image-tests                    %{buildroot}%{_libexecdir}/osbuild-composer-test/
 install -m 0755 -vp _bin/osbuild-auth-tests                     %{buildroot}%{_libexecdir}/osbuild-composer-test/
 install -m 0755 -vp _bin/osbuild-koji-tests                     %{buildroot}%{_libexecdir}/osbuild-composer-test/
+install -m 0755 -vp _bin/osbuild-composer-dbjobqueue-tests      %{buildroot}%{_libexecdir}/osbuild-composer-test/
 install -m 0755 -vp _bin/cloud-cleaner                          %{buildroot}%{_libexecdir}/osbuild-composer-test/
 install -m 0755 -vp tools/define-compose-url.sh                 %{buildroot}%{_libexecdir}/osbuild-composer-test/
 install -m 0755 -vp tools/provision.sh                          %{buildroot}%{_libexecdir}/osbuild-composer-test/
 install -m 0755 -vp tools/gen-certs.sh                          %{buildroot}%{_libexecdir}/osbuild-composer-test/
+install -m 0755 -vp tools/gen-ssh.sh                            %{buildroot}%{_libexecdir}/osbuild-composer-test/
 install -m 0755 -vp tools/image-info                            %{buildroot}%{_libexecdir}/osbuild-composer-test/
 install -m 0755 -vp tools/run-koji-container.sh                 %{buildroot}%{_libexecdir}/osbuild-composer-test/
 install -m 0755 -vp tools/koji-compose.py                       %{buildroot}%{_libexecdir}/osbuild-composer-test/
@@ -235,6 +226,9 @@ install -m 0644 -vp test/data/koji/*                            %{buildroot}%{_d
 install -m 0755 -vd                                             %{buildroot}%{_datadir}/tests/osbuild-composer/x509
 install -m 0644 -vp test/data/x509/*                            %{buildroot}%{_datadir}/tests/osbuild-composer/x509/
 
+install -m 0755 -vd                                             %{buildroot}%{_datadir}/tests/osbuild-composer/schemas
+install -m 0644 -vp internal/jobqueue/dbjobqueue/schemas/*      %{buildroot}%{_datadir}/tests/osbuild-composer/schemas/
+
 %endif
 
 %check
@@ -283,8 +277,8 @@ The core osbuild-composer binary. This is suitable both for spawning in containe
 Summary:    The worker for osbuild-composer
 Requires:   systemd
 Requires:   qemu-img
-Requires:   osbuild >= 28
+Requires:   osbuild >= 29
-Requires:   osbuild-ostree >= 28
+Requires:   osbuild-ostree >= 29
 
 # remove in F34
 Obsoletes: golang-github-osbuild-composer-worker < %{version}-%{release}
@@ -350,6 +344,7 @@ Requires:   virt-install
 Requires:   expect
 Requires:   python3-lxml
 Requires:   httpd
+Requires:   mod_ssl
 Requires:   openssl
 Requires:   podman-plugins
 Requires:   dnf-plugins-core
@@ -376,6 +371,9 @@ Integration tests to be run on a pristine-dedicated system to test the osbuild-c
 %endif
 
 %changelog
+* Sun Aug 15 2021 Ondřej Budai <ondrej@budai.cz> - 31-1
+- New upstream release
+
 * Mon Aug 09 2021 Mohan Boddu <mboddu@redhat.com> - 30-2
 - Rebuilt for IMA sigs, glibc 2.34, aarch64 flags
   Related: rhbz#1991688

sources

@@ -1 +1 @@
-SHA512 (osbuild-composer-30.tar.gz) = 2b2f56d29b61a242edc1ef79502862dbb651e23ba9f717908793401352bd4cf0ec17229df4643e40a42cc4740506adf044fac05565d95d1e64e4c0aee1ae7be9
+SHA512 (osbuild-composer-31.tar.gz) = 4a1acd5f131587f73c91f4185ec7c229ad47de613302aaf792916f49ead4643dcb58482b66a730e86b9040711fef589e2d113c7e21a72772cee4ca63e4bba276

tests/scripts/run_tests.sh

@@ -1,52 +1,15 @@
 #!/bin/bash
 set -euxo pipefail
 
-TOPDIR=$1
-
 # Print some information about the machine
 df -h
 free -h
 
-# osbuild-composer-tests requires koji and ansible packages not available in
+# Tests for osbuild-composer on RHEL 9 fail for multiple reasons,
-# RHEL. Let's enable EPEL and install them.
+# see: https://github.com/osbuild/osbuild-composer/issues?q=is%3Aopen+is%3Aissue+label%3Arhel9
-dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
-dnf repolist -v
-dnf config-manager --enable epel
-dnf repolist -v
-dnf install -y koji ansible
 
-# osbuild-composer-tests contains repo overrides for testing. Override the
+# Let's run only image tests for now.
-# overrides with latest compose. (We don't want to rely on latest compose
-# upstream as it randomly breaks the CI.)
-cp $TOPDIR/repositories/* /usr/share/tests/osbuild-composer/repositories/
-
-# The tests put logs in WORKSPACE, create a temporary dir for this purpose.
-# See https://github.com/osbuild/osbuild-composer/issues/1096
-mkdir /tmp/osbuild-composer-tests
-export WORKSPACE=/tmp/osbuild-composer-tests
-
-# Check that there are no new test executables.
-[ $(find /usr/libexec/tests/osbuild-composer/ -maxdepth 1 -type f | wc -l) -eq "9" ] || exit 5
-
-# In a perfect world, we should be able to run all test executables just by
-# iterating over them. Sadly, this is not possible, see the comments.
-/usr/libexec/tests/osbuild-composer/base_tests.sh
-/usr/libexec/tests/osbuild-composer/koji.sh
-/usr/libexec/tests/osbuild-composer/libvirt.sh
-
-# /usr/libexec/tests/osbuild-composer/image_tests.sh requires nested virt
-# not available in OSCI and it also doesn't support turning off the booting
-# test. Work around it by calling the underlying executable directly as
-# it supports the -disable-local-boot argument.
 /usr/libexec/osbuild-composer-test/osbuild-image-tests \
     -disable-local-boot -fail-local-boot=false \
-    /usr/share/tests/osbuild-composer/manifests/rhel_84-*
+    -skip-selinux-ctx-check \
-
+    /usr/share/tests/osbuild-composer/manifests/rhel_90-*
-# tests not invoked:
-# - ostree.sh    - sadly, virt-install for RHEL 8 creates a VM with 3 GB
-#                  of memory. That much memory isn't available in OSCI.
-#                  Also, missing nested virt support is a big issue.
-# - ostree-ng.sh - Same as above.
-# - api.sh       - requires secrets, we cannot pass them safely in OSCI.
-# - aws.sh       - requires secrets, we cannot pass them safely in OSCI.
-# - vmware.sh    - requires secrets, we cannot pass them safely in OSCI.