Update to 31

This commit also enables at least some meaningful tests in gating.

Resolves: rhbz#1972427
This commit is contained in:
Ondřej Budai 2021-08-15 08:00:35 +02:00
parent b612db2122
commit faa4908d14
No known key found for this signature in database
GPG Key ID: 2EDC02DB69E9C992
7 changed files with 10895 additions and 64 deletions

1
.gitignore vendored
View File

@ -20,3 +20,4 @@
/osbuild-composer-28.tar.gz /osbuild-composer-28.tar.gz
/osbuild-composer-29.tar.gz /osbuild-composer-29.tar.gz
/osbuild-composer-30.tar.gz /osbuild-composer-30.tar.gz
/osbuild-composer-31.tar.gz

View File

@ -0,0 +1,75 @@
From 85cc7687415a96db017acaf763d53abbc47d993f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ond=C5=99ej=20Budai?= <ondrej@budai.cz>
Date: Mon, 16 Aug 2021 12:56:55 +0200
Subject: [PATCH] test/image: fix pipeline exports for v2 manifests
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Previously, we just assumed that all test manifests are of version 1, or we
should export the pipeline named assembler. However, this is no longer true
in RHEL 8.5 and 9 - they are only manifest v2 and they don't have a pipeline
named assembler.
This commit introduces a new way to guess the export name - if the manifest
is of version 1, we just export the assembler. In the case v2 manifest, the
last pipeline is exported.
Signed-off-by: Ondřej Budai <ondrej@budai.cz>
---
cmd/osbuild-image-tests/main_test.go | 32 +++++++++++++++++++++++++---
1 file changed, 29 insertions(+), 3 deletions(-)
diff --git a/cmd/osbuild-image-tests/main_test.go b/cmd/osbuild-image-tests/main_test.go
index f6cbccc9..f73bab3d 100644
--- a/cmd/osbuild-image-tests/main_test.go
+++ b/cmd/osbuild-image-tests/main_test.go
@@ -482,6 +482,34 @@ func testImage(t *testing.T, testcase testcaseStruct, imagePath string) {
}
}
+// guessPipelineToExport return a best-effort guess about which
+// pipeline should be exported when running osbuild for the testcase
+//
+// If this function detects that this is a version 1 manifest, it
+// always returns "assembler"
+//
+// For manifests version 2, the name of the last pipeline is returned.
+func guessPipelineToExport(rawManifest json.RawMessage) string {
+ const v1ManifestExportName = "assembler"
+ var v2Manifest struct {
+ Version string `json:"version"`
+ Pipelines []struct {
+ Name string `json:"name,omitempty"`
+ } `json:"pipelines"`
+ }
+ err := json.Unmarshal(rawManifest, &v2Manifest)
+ if err != nil {
+ // if we cannot unmarshal, let's just assume that it's a version 1 manifest
+ return v1ManifestExportName
+ }
+
+ if v2Manifest.Version == "2" {
+ return v2Manifest.Pipelines[len(v2Manifest.Pipelines)-1].Name
+ }
+
+ return v1ManifestExportName
+}
+
// runTestcase builds the pipeline specified in the testcase and then it
// tests the result
func runTestcase(t *testing.T, testcase testcaseStruct, store string) {
@@ -494,9 +522,7 @@ func runTestcase(t *testing.T, testcase testcaseStruct, store string) {
require.NoError(t, err, "error removing temporary output directory")
}()
- // NOTE(akoutsou) 1to2t: new v2 manifests name their last pipeline
- // "assembler" for compatibility with v1
- exports := []string{"assembler"}
+ exports := []string{guessPipelineToExport(testcase.Manifest)}
err = runOsbuild(testcase.Manifest, store, outputDirectory, exports)
require.NoError(t, err)
--
2.31.1

10794
0002-PR-1638-fix-ami.patch Normal file

File diff suppressed because it is too large Load Diff

View File

@ -4,7 +4,7 @@
%global goipath github.com/osbuild/osbuild-composer %global goipath github.com/osbuild/osbuild-composer
Version: 30 Version: 31
%gometa %gometa
@ -17,7 +17,7 @@ It is compatible with composer-cli and cockpit-composer clients.
} }
Name: osbuild-composer Name: osbuild-composer
Release: 2%{?dist} Release: 1%{?dist}
Summary: An image building service based on osbuild Summary: An image building service based on osbuild
# osbuild-composer doesn't have support for building i686 images # osbuild-composer doesn't have support for building i686 images
@ -28,7 +28,8 @@ ExcludeArch: i686
License: ASL 2.0 License: ASL 2.0
URL: %{gourl} URL: %{gourl}
Source0: %{gosource} Source0: %{gosource}
Patch0: 0001-PR-1645-test-image-fix-pipeline-exports-for-v2-manifests.patch
Patch1: 0002-PR-1638-fix-ami.patch
BuildRequires: %{?go_compiler:compiler(go-compiler)}%{!?go_compiler:golang} BuildRequires: %{?go_compiler:compiler(go-compiler)}%{!?go_compiler:golang}
BuildRequires: systemd BuildRequires: systemd
@ -47,6 +48,7 @@ BuildRequires: golang(github.com/coreos/go-systemd/activation)
BuildRequires: golang(github.com/deepmap/oapi-codegen/pkg/codegen) BuildRequires: golang(github.com/deepmap/oapi-codegen/pkg/codegen)
BuildRequires: golang(github.com/go-chi/chi) BuildRequires: golang(github.com/go-chi/chi)
BuildRequires: golang(github.com/google/uuid) BuildRequires: golang(github.com/google/uuid)
BuildRequires: golang(github.com/jackc/pgx/v4)
BuildRequires: golang(github.com/julienschmidt/httprouter) BuildRequires: golang(github.com/julienschmidt/httprouter)
BuildRequires: golang(github.com/getkin/kin-openapi/openapi3) BuildRequires: golang(github.com/getkin/kin-openapi/openapi3)
BuildRequires: golang(github.com/kolo/xmlrpc) BuildRequires: golang(github.com/kolo/xmlrpc)
@ -54,10 +56,12 @@ BuildRequires: golang(github.com/labstack/echo/v4)
BuildRequires: golang(github.com/gobwas/glob) BuildRequires: golang(github.com/gobwas/glob)
BuildRequires: golang(github.com/google/go-cmp/cmp) BuildRequires: golang(github.com/google/go-cmp/cmp)
BuildRequires: golang(github.com/gophercloud/gophercloud) BuildRequires: golang(github.com/gophercloud/gophercloud)
BuildRequires: golang(github.com/prometheus/client_golang/prometheus/promhttp)
BuildRequires: golang(github.com/stretchr/testify/assert) BuildRequires: golang(github.com/stretchr/testify/assert)
BuildRequires: golang(github.com/ubccr/kerby) BuildRequires: golang(github.com/ubccr/kerby)
BuildRequires: golang(github.com/vmware/govmomi) BuildRequires: golang(github.com/vmware/govmomi)
BuildRequires: golang(cloud.google.com/go) BuildRequires: golang(cloud.google.com/go)
BuildRequires: golang(gopkg.in/ini.v1)
%endif %endif
Requires: %{name}-core = %{version}-%{release} Requires: %{name}-core = %{version}-%{release}
@ -94,22 +98,6 @@ Obsoletes: osbuild-composer-koji <= 23
%goprep %goprep
%endif %endif
%if 0%{?fedora} && 0%{?fedora} <= 32
# Fedora 32 and older ships different kolo/xmlrpc and azure/azblob APIs. We
# cannot specify build tags in gobuild macro because the macro itself
# specifies build tags and -tags argument cannot be used more than once.
# Therefore, this ugly hack with build tags switcharoo is required.
# Remove when F32 is EOL.
# Remove the build constraint from the wrappers of the old APIs
sed -i "s$// +build kolo_xmlrpc_oldapi$// +build !kolo_xmlrpc_oldapi$" internal/upload/koji/xmlrpc-response-oldapi.go
sed -i "s$// +build azblob_oldapi$// +build !azblob_oldapi$" internal/upload/azure/page_blob_url_oldapi.go
# Add a build constraint to the wrappers of the new APIs
sed -i "s$// +build !kolo_xmlrpc_oldapi$// +build kolo_xmlrpc_oldapi$" internal/upload/koji/xmlrpc-response.go
sed -i "s$// +build !azblob_oldapi$// +build azblob_oldapi$" internal/upload/azure/page_blob_url.go
%endif
%if 0%{?fedora} >= 34 %if 0%{?fedora} >= 34
# Fedora 34 and newer ships a newer version of github.com/getkin/kin-openapi # Fedora 34 and newer ships a newer version of github.com/getkin/kin-openapi
# package which has a different API than the older ones. Let's make the auto- # package which has a different API than the older ones. Let's make the auto-
@ -157,6 +145,7 @@ go test -c -tags=integration -ldflags="${TEST_LDFLAGS}" -o _bin/osbuild-weldr-te
go test -c -tags=integration -ldflags="${TEST_LDFLAGS}" -o _bin/osbuild-image-tests %{goipath}/cmd/osbuild-image-tests go test -c -tags=integration -ldflags="${TEST_LDFLAGS}" -o _bin/osbuild-image-tests %{goipath}/cmd/osbuild-image-tests
go test -c -tags=integration -ldflags="${TEST_LDFLAGS}" -o _bin/osbuild-auth-tests %{goipath}/cmd/osbuild-auth-tests go test -c -tags=integration -ldflags="${TEST_LDFLAGS}" -o _bin/osbuild-auth-tests %{goipath}/cmd/osbuild-auth-tests
go test -c -tags=integration -ldflags="${TEST_LDFLAGS}" -o _bin/osbuild-koji-tests %{goipath}/cmd/osbuild-koji-tests go test -c -tags=integration -ldflags="${TEST_LDFLAGS}" -o _bin/osbuild-koji-tests %{goipath}/cmd/osbuild-koji-tests
go test -c -tags=integration -ldflags="${TEST_LDFLAGS}" -o _bin/osbuild-composer-dbjobqueue-tests %{goipath}/cmd/osbuild-composer-dbjobqueue-tests
go build -tags=integration -ldflags="${TEST_LDFLAGS}" -o _bin/cloud-cleaner %{goipath}/cmd/cloud-cleaner go build -tags=integration -ldflags="${TEST_LDFLAGS}" -o _bin/cloud-cleaner %{goipath}/cmd/cloud-cleaner
%endif %endif
@ -190,10 +179,12 @@ install -m 0755 -vp _bin/osbuild-dnf-json-tests %{buildroot}%{_l
install -m 0755 -vp _bin/osbuild-image-tests %{buildroot}%{_libexecdir}/osbuild-composer-test/ install -m 0755 -vp _bin/osbuild-image-tests %{buildroot}%{_libexecdir}/osbuild-composer-test/
install -m 0755 -vp _bin/osbuild-auth-tests %{buildroot}%{_libexecdir}/osbuild-composer-test/ install -m 0755 -vp _bin/osbuild-auth-tests %{buildroot}%{_libexecdir}/osbuild-composer-test/
install -m 0755 -vp _bin/osbuild-koji-tests %{buildroot}%{_libexecdir}/osbuild-composer-test/ install -m 0755 -vp _bin/osbuild-koji-tests %{buildroot}%{_libexecdir}/osbuild-composer-test/
install -m 0755 -vp _bin/osbuild-composer-dbjobqueue-tests %{buildroot}%{_libexecdir}/osbuild-composer-test/
install -m 0755 -vp _bin/cloud-cleaner %{buildroot}%{_libexecdir}/osbuild-composer-test/ install -m 0755 -vp _bin/cloud-cleaner %{buildroot}%{_libexecdir}/osbuild-composer-test/
install -m 0755 -vp tools/define-compose-url.sh %{buildroot}%{_libexecdir}/osbuild-composer-test/ install -m 0755 -vp tools/define-compose-url.sh %{buildroot}%{_libexecdir}/osbuild-composer-test/
install -m 0755 -vp tools/provision.sh %{buildroot}%{_libexecdir}/osbuild-composer-test/ install -m 0755 -vp tools/provision.sh %{buildroot}%{_libexecdir}/osbuild-composer-test/
install -m 0755 -vp tools/gen-certs.sh %{buildroot}%{_libexecdir}/osbuild-composer-test/ install -m 0755 -vp tools/gen-certs.sh %{buildroot}%{_libexecdir}/osbuild-composer-test/
install -m 0755 -vp tools/gen-ssh.sh %{buildroot}%{_libexecdir}/osbuild-composer-test/
install -m 0755 -vp tools/image-info %{buildroot}%{_libexecdir}/osbuild-composer-test/ install -m 0755 -vp tools/image-info %{buildroot}%{_libexecdir}/osbuild-composer-test/
install -m 0755 -vp tools/run-koji-container.sh %{buildroot}%{_libexecdir}/osbuild-composer-test/ install -m 0755 -vp tools/run-koji-container.sh %{buildroot}%{_libexecdir}/osbuild-composer-test/
install -m 0755 -vp tools/koji-compose.py %{buildroot}%{_libexecdir}/osbuild-composer-test/ install -m 0755 -vp tools/koji-compose.py %{buildroot}%{_libexecdir}/osbuild-composer-test/
@ -235,6 +226,9 @@ install -m 0644 -vp test/data/koji/* %{buildroot}%{_d
install -m 0755 -vd %{buildroot}%{_datadir}/tests/osbuild-composer/x509 install -m 0755 -vd %{buildroot}%{_datadir}/tests/osbuild-composer/x509
install -m 0644 -vp test/data/x509/* %{buildroot}%{_datadir}/tests/osbuild-composer/x509/ install -m 0644 -vp test/data/x509/* %{buildroot}%{_datadir}/tests/osbuild-composer/x509/
install -m 0755 -vd %{buildroot}%{_datadir}/tests/osbuild-composer/schemas
install -m 0644 -vp internal/jobqueue/dbjobqueue/schemas/* %{buildroot}%{_datadir}/tests/osbuild-composer/schemas/
%endif %endif
%check %check
@ -283,8 +277,8 @@ The core osbuild-composer binary. This is suitable both for spawning in containe
Summary: The worker for osbuild-composer Summary: The worker for osbuild-composer
Requires: systemd Requires: systemd
Requires: qemu-img Requires: qemu-img
Requires: osbuild >= 28 Requires: osbuild >= 29
Requires: osbuild-ostree >= 28 Requires: osbuild-ostree >= 29
# remove in F34 # remove in F34
Obsoletes: golang-github-osbuild-composer-worker < %{version}-%{release} Obsoletes: golang-github-osbuild-composer-worker < %{version}-%{release}
@ -350,6 +344,7 @@ Requires: virt-install
Requires: expect Requires: expect
Requires: python3-lxml Requires: python3-lxml
Requires: httpd Requires: httpd
Requires: mod_ssl
Requires: openssl Requires: openssl
Requires: podman-plugins Requires: podman-plugins
Requires: dnf-plugins-core Requires: dnf-plugins-core
@ -376,6 +371,9 @@ Integration tests to be run on a pristine-dedicated system to test the osbuild-c
%endif %endif
%changelog %changelog
* Sun Aug 15 2021 Ondřej Budai <ondrej@budai.cz> - 31-1
- New upstream release
* Mon Aug 09 2021 Mohan Boddu <mboddu@redhat.com> - 30-2 * Mon Aug 09 2021 Mohan Boddu <mboddu@redhat.com> - 30-2
- Rebuilt for IMA sigs, glibc 2.34, aarch64 flags - Rebuilt for IMA sigs, glibc 2.34, aarch64 flags
Related: rhbz#1991688 Related: rhbz#1991688

View File

@ -1 +1 @@
SHA512 (osbuild-composer-30.tar.gz) = 2b2f56d29b61a242edc1ef79502862dbb651e23ba9f717908793401352bd4cf0ec17229df4643e40a42cc4740506adf044fac05565d95d1e64e4c0aee1ae7be9 SHA512 (osbuild-composer-31.tar.gz) = 4a1acd5f131587f73c91f4185ec7c229ad47de613302aaf792916f49ead4643dcb58482b66a730e86b9040711fef589e2d113c7e21a72772cee4ca63e4bba276

View File

@ -1,52 +1,15 @@
#!/bin/bash #!/bin/bash
set -euxo pipefail set -euxo pipefail
TOPDIR=$1
# Print some information about the machine # Print some information about the machine
df -h df -h
free -h free -h
# osbuild-composer-tests requires koji and ansible packages not available in # Tests for osbuild-composer on RHEL 9 fail for multiple reasons,
# RHEL. Let's enable EPEL and install them. # see: https://github.com/osbuild/osbuild-composer/issues?q=is%3Aopen+is%3Aissue+label%3Arhel9
dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
dnf repolist -v
dnf config-manager --enable epel
dnf repolist -v
dnf install -y koji ansible
# osbuild-composer-tests contains repo overrides for testing. Override the # Let's run only image tests for now.
# overrides with latest compose. (We don't want to rely on latest compose
# upstream as it randomly breaks the CI.)
cp $TOPDIR/repositories/* /usr/share/tests/osbuild-composer/repositories/
# The tests put logs in WORKSPACE, create a temporary dir for this purpose.
# See https://github.com/osbuild/osbuild-composer/issues/1096
mkdir /tmp/osbuild-composer-tests
export WORKSPACE=/tmp/osbuild-composer-tests
# Check that there are no new test executables.
[ $(find /usr/libexec/tests/osbuild-composer/ -maxdepth 1 -type f | wc -l) -eq "9" ] || exit 5
# In a perfect world, we should be able to run all test executables just by
# iterating over them. Sadly, this is not possible, see the comments.
/usr/libexec/tests/osbuild-composer/base_tests.sh
/usr/libexec/tests/osbuild-composer/koji.sh
/usr/libexec/tests/osbuild-composer/libvirt.sh
# /usr/libexec/tests/osbuild-composer/image_tests.sh requires nested virt
# not available in OSCI and it also doesn't support turning off the booting
# test. Work around it by calling the underlying executable directly as
# it supports the -disable-local-boot argument.
/usr/libexec/osbuild-composer-test/osbuild-image-tests \ /usr/libexec/osbuild-composer-test/osbuild-image-tests \
-disable-local-boot -fail-local-boot=false \ -disable-local-boot -fail-local-boot=false \
/usr/share/tests/osbuild-composer/manifests/rhel_84-* -skip-selinux-ctx-check \
/usr/share/tests/osbuild-composer/manifests/rhel_90-*
# tests not invoked:
# - ostree.sh - sadly, virt-install for RHEL 8 creates a VM with 3 GB
# of memory. That much memory isn't available in OSCI.
# Also, missing nested virt support is a big issue.
# - ostree-ng.sh - Same as above.
# - api.sh - requires secrets, we cannot pass them safely in OSCI.
# - aws.sh - requires secrets, we cannot pass them safely in OSCI.
# - vmware.sh - requires secrets, we cannot pass them safely in OSCI.