diff --git a/0001-image-build-Expand-arches-for-can_fail.patch b/0001-image-build-Expand-arches-for-can_fail.patch new file mode 100644 index 00000000..3ac6ea91 --- /dev/null +++ b/0001-image-build-Expand-arches-for-can_fail.patch @@ -0,0 +1,104 @@ +From 6c708549c8f6632884bc55abed2f88afa1abe100 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= +Date: Mon, 23 Jan 2017 16:52:57 +0100 +Subject: [PATCH 1/2] image-build: Expand arches for can_fail +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +We need to work with a list of strings, not a comma-delimited single +string. + +Signed-off-by: Lubomír Sedlář +--- + pungi/phases/image_build.py | 2 +- + tests/test_imagebuildphase.py | 60 +++++++++++++++++++++++++++++++++++++++++++ + 2 files changed, 61 insertions(+), 1 deletion(-) + +diff --git a/pungi/phases/image_build.py b/pungi/phases/image_build.py +index 5cfdbb0..b795125 100644 +--- a/pungi/phases/image_build.py ++++ b/pungi/phases/image_build.py +@@ -122,7 +122,7 @@ class ImageBuildPhase(base.PhaseLoggerMixin, base.ImageConfigMixin, base.ConfigG + + can_fail = image_conf['image-build'].pop('failable', []) + if can_fail == ['*']: +- can_fail = image_conf['image-build']['arches'] ++ can_fail = image_conf['image-build']['arches'].split(',') + if can_fail: + image_conf['image-build']['can_fail'] = ','.join(sorted(can_fail)) + +diff --git a/tests/test_imagebuildphase.py b/tests/test_imagebuildphase.py +index 86012ac..328b2b3 100644 +--- a/tests/test_imagebuildphase.py ++++ b/tests/test_imagebuildphase.py +@@ -619,6 +619,66 @@ class TestImageBuildPhase(PungiTestCase): + self.assertItemsEqual(phase.pool.queue_put.mock_calls, + [mock.call((compose, server_args))]) + ++ @mock.patch('pungi.phases.image_build.ThreadPool') ++ def test_failable_star(self, ThreadPool): ++ compose = DummyCompose(self.topdir, { ++ 'image_build': { ++ '^Server$': [ ++ { ++ 'image-build': { ++ 'format': [('docker', 'tar.xz')], ++ 'name': 'Fedora-Docker-Base', ++ 'target': 'f24', ++ 'version': 'Rawhide', ++ 'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', ++ 'kickstart': "fedora-docker-base.ks", ++ 'distro': 'Fedora-20', ++ 'disk_size': 3, ++ 'failable': ['*'], ++ } ++ } ++ ] ++ }, ++ 'koji_profile': 'koji', ++ }) ++ compose.setup_optional() ++ ++ self.assertValidConfig(compose.conf) ++ ++ phase = ImageBuildPhase(compose) ++ ++ phase.run() ++ ++ # assert at least one thread was started ++ self.assertTrue(phase.pool.add.called) ++ server_args = { ++ "format": [('docker', 'tar.xz')], ++ "image_conf": { ++ 'image-build': { ++ 'install_tree': self.topdir + '/compose/Server/$arch/os', ++ 'kickstart': 'fedora-docker-base.ks', ++ 'format': 'docker', ++ 'repo': self.topdir + '/compose/Server/$arch/os', ++ 'variant': compose.all_variants['Server'], ++ 'target': 'f24', ++ 'disk_size': 3, ++ 'name': 'Fedora-Docker-Base', ++ 'arches': 'amd64,x86_64', ++ 'version': 'Rawhide', ++ 'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', ++ 'distro': 'Fedora-20', ++ 'can_fail': 'amd64,x86_64', ++ } ++ }, ++ "conf_file": self.topdir + '/work/image-build/Server/docker_Fedora-Docker-Base.cfg', ++ "image_dir": self.topdir + '/compose/Server/%(arch)s/images', ++ "relative_image_dir": 'Server/%(arch)s/images', ++ "link_type": 'hardlink-or-copy', ++ "scratch": False, ++ } ++ self.assertItemsEqual(phase.pool.queue_put.mock_calls, ++ [mock.call((compose, server_args))]) ++ + + class TestCreateImageBuildThread(PungiTestCase): + +-- +2.11.0 + diff --git a/0001-replace-basearch-when-updating-the-ref.patch b/0001-replace-basearch-when-updating-the-ref.patch index 5e4a4ebb..4f44cae1 100644 --- a/0001-replace-basearch-when-updating-the-ref.patch +++ b/0001-replace-basearch-when-updating-the-ref.patch @@ -1,38 +1,66 @@ -From 1e9962be27718171e9291c1b62983229ca49b907 Mon Sep 17 00:00:00 2001 +From 59dd4dbcd84ce30faa6558ad0d052370077d3fe5 Mon Sep 17 00:00:00 2001 From: Dennis Gilmore -Date: Wed, 18 Jan 2017 14:22:43 -0600 -Subject: [PATCH] replace ${basearch} when updating the ref +Date: Jan 19 2017 03:21:26 +0000 +Subject: replace ${basearch} when updating the ref + pungi-make-ostree has to run on the target arch so that rpm -scriptlets can be ran.as a reult we can ask rpm what the -basearch is for the running environment +scriptlets can be ran. as a reult we can ask rpm what the +basearch is for the running environment. For notifications +we have to pass in the arch we are running for. Signed-off-by: Dennis Gilmore ---- - pungi/ostree/tree.py | 3 +++ - 1 file changed, 3 insertions(+) -diff --git a/pungi/ostree/tree.py b/pungi/ostree/tree.py -index 68b6d70..94a0be2 100644 ---- a/pungi/ostree/tree.py -+++ b/pungi/ostree/tree.py -@@ -16,6 +16,7 @@ - - import os +--- + +diff --git a/pungi/ostree/utils.py b/pungi/ostree/utils.py +index 02540c9..d80fb9f 100644 +--- a/pungi/ostree/utils.py ++++ b/pungi/ostree/utils.py +@@ -17,6 +17,7 @@ + import datetime import json + import os +import rpmUtils.arch - from kobo import shortcuts from pungi.util import makedirs -@@ -67,6 +68,8 @@ class Tree(OSTree): - return - ref = get_ref_from_treefile(self.treefile) - commitid = get_commitid_from_commitid_file(self.commitid_file) -+ if ref is not None: -+ ref = ref.replace('${basearch}', rpmUtils.arch.getBaseArch()) - print('Ref: %r, Commit ID: %r' % (ref, commitid)) - if ref and commitid: - print('Updating ref') --- -2.11.0 + +@@ -29,14 +30,21 @@ def make_log_file(log_dir, filename): + return os.path.join(log_dir, '%s.log' % filename) + + +-def get_ref_from_treefile(treefile): +- """Return ref name by parsing the tree config file""" ++def get_ref_from_treefile(treefile, arch=None): ++ """ ++ Return ref name by parsing the tree config file. Replacing ${basearch} with ++ the basearch of the architecture we are running on or of the passed in arch. ++ """ + ref = None + if os.path.isfile(treefile): + with open(treefile, 'r') as f: + try: + parsed = json.loads(f.read()) +- ref = parsed['ref'] ++ if arch is None: ++ basearch = rpmUtils.arch.getBaseArch() ++ else: ++ basearch = rpmUtils.arch.getBaseArch(arch) ++ ref = parsed['ref'].replace('${basearch}', basearch) + except Exception as e: + print('Unable to get ref from treefile: %s' % e) + else: +diff --git a/pungi/phases/ostree.py b/pungi/phases/ostree.py +index ebe03d0..e61e969 100644 +--- a/pungi/phases/ostree.py ++++ b/pungi/phases/ostree.py +@@ -104,7 +104,7 @@ class OSTreeThread(WorkerThread): + extra_config_file=extra_config_file) + + if compose.notifier: +- ref = get_ref_from_treefile(os.path.join(repodir, config['treefile'])) ++ ref = get_ref_from_treefile(os.path.join(repodir, config['treefile']), arch) + # 'pungi-make-ostree tree' writes commitid to commitid.log in logdir + commitid = get_commitid_from_commitid_file(os.path.join(self.logdir, 'commitid.log')) + compose.notifier.send('ostree', diff --git a/0002-image-build-Pass-arches-around-as-a-list.patch b/0002-image-build-Pass-arches-around-as-a-list.patch new file mode 100644 index 00000000..b86429b9 --- /dev/null +++ b/0002-image-build-Pass-arches-around-as-a-list.patch @@ -0,0 +1,241 @@ +From 8418b68fb0e1253f688b97c91df9ace6a38ebd02 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= +Date: Tue, 24 Jan 2017 08:36:33 +0100 +Subject: [PATCH 2/2] image-build: Pass arches around as a list +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Instead of joining the arches as a comma separated string and splitting +it again later. Ultimately we do need the original format to pass to +koji wrapper, but we can produce that value later. + +Signed-off-by: Lubomír Sedlář +--- + pungi/phases/image_build.py | 16 ++++++++++------ + tests/test_imagebuildphase.py | 41 +++++++++++++++++++++-------------------- + 2 files changed, 31 insertions(+), 26 deletions(-) + +diff --git a/pungi/phases/image_build.py b/pungi/phases/image_build.py +index b795125..bd18d3a 100644 +--- a/pungi/phases/image_build.py ++++ b/pungi/phases/image_build.py +@@ -75,7 +75,7 @@ class ImageBuildPhase(base.PhaseLoggerMixin, base.ImageConfigMixin, base.ConfigG + def _get_arches(self, image_conf, arches): + if 'arches' in image_conf['image-build']: + arches = set(image_conf['image-build'].get('arches', [])) & arches +- return ','.join(sorted(arches)) ++ return sorted(arches) + + def _set_release(self, image_conf): + """If release is set explicitly to None, replace it with date and respin.""" +@@ -122,9 +122,9 @@ class ImageBuildPhase(base.PhaseLoggerMixin, base.ImageConfigMixin, base.ConfigG + + can_fail = image_conf['image-build'].pop('failable', []) + if can_fail == ['*']: +- can_fail = image_conf['image-build']['arches'].split(',') ++ can_fail = image_conf['image-build']['arches'] + if can_fail: +- image_conf['image-build']['can_fail'] = ','.join(sorted(can_fail)) ++ image_conf['image-build']['can_fail'] = sorted(can_fail) + + cmd = { + "format": format, +@@ -162,8 +162,7 @@ class CreateImageBuildThread(WorkerThread): + self.worker(num, compose, variant, subvariant, cmd) + + def worker(self, num, compose, variant, subvariant, cmd): +- arches = cmd["image_conf"]["image-build"]['arches'].split(',') +- failable_arches = self.failable_arches.split(',') ++ arches = cmd["image_conf"]["image-build"]['arches'] + dash_arches = '-'.join(arches) + log_file = compose.paths.log.log_file( + dash_arches, +@@ -180,6 +179,11 @@ class CreateImageBuildThread(WorkerThread): + # writes conf file for koji image-build + self.pool.log_info("Writing image-build config for %s.%s into %s" % ( + variant, dash_arches, cmd["conf_file"])) ++ ++ # Join the arches into a single string. This is the value expected by ++ # koji config file. ++ cmd["image_conf"]["image-build"]['arches'] = ','.join(cmd["image_conf"]["image-build"]['arches']) ++ + koji_cmd = koji_wrapper.get_image_build_cmd(cmd["image_conf"], + conf_file_dest=cmd["conf_file"], + scratch=cmd['scratch']) +@@ -207,7 +211,7 @@ class CreateImageBuildThread(WorkerThread): + image_infos.append({'path': path, 'suffix': suffix, 'type': format, 'arch': arch}) + break + +- if len(image_infos) != len(cmd['format']) * (len(arches) - len(failable_arches)): ++ if len(image_infos) != len(cmd['format']) * (len(arches) - len(self.failable_arches)): + self.pool.log_error( + "Error in koji task %s. Expected to find same amount of images " + "as in suffixes attr in image-build (%s) for each arch (%s). Got '%s'." % +diff --git a/tests/test_imagebuildphase.py b/tests/test_imagebuildphase.py +index 328b2b3..d3185e2 100644 +--- a/tests/test_imagebuildphase.py ++++ b/tests/test_imagebuildphase.py +@@ -66,11 +66,11 @@ class TestImageBuildPhase(PungiTestCase): + 'target': 'f24', + 'disk_size': 3, + 'name': 'Fedora-Docker-Base', +- 'arches': 'amd64', ++ 'arches': ['amd64'], + 'version': 'Rawhide', + 'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', + 'distro': 'Fedora-20', +- 'can_fail': 'x86_64', ++ 'can_fail': ['x86_64'], + } + }, + "conf_file": self.topdir + '/work/image-build/Client/docker_Fedora-Docker-Base.cfg', +@@ -91,11 +91,11 @@ class TestImageBuildPhase(PungiTestCase): + 'target': 'f24', + 'disk_size': 3, + 'name': 'Fedora-Docker-Base', +- 'arches': 'amd64,x86_64', ++ 'arches': ['amd64', 'x86_64'], + 'version': 'Rawhide', + 'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', + 'distro': 'Fedora-20', +- 'can_fail': 'x86_64', ++ 'can_fail': ['x86_64'], + } + }, + "conf_file": self.topdir + '/work/image-build/Server/docker_Fedora-Docker-Base.cfg', +@@ -151,7 +151,7 @@ class TestImageBuildPhase(PungiTestCase): + 'target': 'f24', + 'disk_size': 3, + 'name': 'Fedora-Docker-Base', +- 'arches': 'amd64,x86_64', ++ 'arches': ['amd64', 'x86_64'], + 'version': 'Rawhide', + 'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', + 'distro': 'Fedora-20', +@@ -207,7 +207,7 @@ class TestImageBuildPhase(PungiTestCase): + 'target': 'f24', + 'disk_size': 3, + 'name': 'Fedora-Docker-Base', +- 'arches': 'amd64,x86_64', ++ 'arches': ['amd64', 'x86_64'], + 'version': '25', + 'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', + 'distro': 'Fedora-20', +@@ -305,7 +305,7 @@ class TestImageBuildPhase(PungiTestCase): + 'target': 'f24', + 'disk_size': 3, + 'name': 'Fedora-Docker-Base', +- 'arches': 'x86_64', ++ 'arches': ['x86_64'], + 'version': 'Rawhide', + 'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', + 'distro': 'Fedora-20', +@@ -369,7 +369,7 @@ class TestImageBuildPhase(PungiTestCase): + 'target': 'f24', + 'disk_size': 3, + 'name': 'Fedora-Docker-Base', +- 'arches': 'x86_64', ++ 'arches': ['x86_64'], + 'version': 'Rawhide', + 'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', + 'distro': 'Fedora-20', +@@ -430,7 +430,7 @@ class TestImageBuildPhase(PungiTestCase): + 'target': 'f24', + 'disk_size': 3, + 'name': 'Fedora-Docker-Base', +- 'arches': 'x86_64', ++ 'arches': ['x86_64'], + 'version': 'Rawhide', + 'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', + 'distro': 'Fedora-20', +@@ -603,11 +603,11 @@ class TestImageBuildPhase(PungiTestCase): + 'target': 'f24', + 'disk_size': 3, + 'name': 'Fedora-Docker-Base', +- 'arches': 'x86_64', ++ 'arches': ['x86_64'], + 'version': 'Rawhide', + 'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', + 'distro': 'Fedora-20', +- 'can_fail': 'x86_64', ++ 'can_fail': ['x86_64'], + } + }, + "conf_file": self.topdir + '/work/image-build/Server-optional/docker_Fedora-Docker-Base.cfg', +@@ -663,11 +663,11 @@ class TestImageBuildPhase(PungiTestCase): + 'target': 'f24', + 'disk_size': 3, + 'name': 'Fedora-Docker-Base', +- 'arches': 'amd64,x86_64', ++ 'arches': ['amd64', 'x86_64'], + 'version': 'Rawhide', + 'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', + 'distro': 'Fedora-20', +- 'can_fail': 'amd64,x86_64', ++ 'can_fail': ['amd64', 'x86_64'], + } + }, + "conf_file": self.topdir + '/work/image-build/Server/docker_Fedora-Docker-Base.cfg', +@@ -703,7 +703,7 @@ class TestCreateImageBuildThread(PungiTestCase): + 'target': 'f24', + 'disk_size': 3, + 'name': 'Fedora-Docker-Base', +- 'arches': 'amd64,x86_64', ++ 'arches': ['amd64', 'x86_64'], + 'version': 'Rawhide', + 'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', + 'distro': 'Fedora-20', +@@ -743,6 +743,7 @@ class TestCreateImageBuildThread(PungiTestCase): + with mock.patch('time.sleep'): + t.process((compose, cmd), 1) + ++ self.assertEqual(cmd['image_conf']['image-build']['arches'], 'amd64,x86_64') + self.assertItemsEqual( + koji_wrapper.get_image_build_cmd.call_args_list, + [mock.call(cmd['image_conf'], +@@ -832,11 +833,11 @@ class TestCreateImageBuildThread(PungiTestCase): + 'target': 'f24', + 'disk_size': 3, + 'name': 'Fedora-Docker-Base', +- 'arches': 'amd64,x86_64', ++ 'arches': ['amd64', 'x86_64'], + 'version': 'Rawhide', + 'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', + 'distro': 'Fedora-20', +- "can_fail": 'amd64,x86_64', ++ "can_fail": ['amd64', 'x86_64'], + } + }, + "conf_file": 'amd64,x86_64-Client-Fedora-Docker-Base-docker', +@@ -880,11 +881,11 @@ class TestCreateImageBuildThread(PungiTestCase): + 'target': 'f24', + 'disk_size': 3, + 'name': 'Fedora-Docker-Base', +- 'arches': 'amd64,x86_64', ++ 'arches': ['amd64', 'x86_64'], + 'version': 'Rawhide', + 'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', + 'distro': 'Fedora-20', +- 'can_fail': 'amd64,x86_64', ++ 'can_fail': ['amd64', 'x86_64'], + } + }, + "conf_file": 'amd64,x86_64-Client-Fedora-Docker-Base-docker', +@@ -923,11 +924,11 @@ class TestCreateImageBuildThread(PungiTestCase): + 'target': 'f24', + 'disk_size': 3, + 'name': 'Fedora-Docker-Base', +- 'arches': 'amd64,x86_64', ++ 'arches': ['amd64', 'x86_64'], + 'version': 'Rawhide', + 'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', + 'distro': 'Fedora-20', +- 'can_fail': 'amd64', ++ 'can_fail': ['amd64'], + } + }, + "conf_file": 'amd64,x86_64-Client-Fedora-Docker-Base-docker', +-- +2.11.0 + diff --git a/pungi.spec b/pungi.spec index 6562ae0c..dfdb30f8 100644 --- a/pungi.spec +++ b/pungi.spec @@ -1,6 +1,6 @@ Name: pungi Version: 4.1.12 -Release: 3%{?dist} +Release: 4%{?dist} Summary: Distribution compose tool Group: Development/Tools @@ -10,6 +10,8 @@ Source0: https://pagure.io/releases/%{name}/%{name}-%{version}.tar.bz2 # from https://pagure.io/fork/puiterwijk/pungi/c/63aa7db5aad4690b72898d01fa82270c1f503241 Patch0: 63aa7db5aad4690b72898d01fa82270c1f503241.patch Patch1: 0001-replace-basearch-when-updating-the-ref.patch +Patch2: 0001-image-build-Expand-arches-for-can_fail.patch +Patch3: 0002-image-build-Pass-arches-around-as-a-list.patch BuildRequires: python-nose, python-mock BuildRequires: python-devel, python-setuptools, python2-productmd >= 1.3 @@ -73,6 +75,8 @@ notification to Fedora Message Bus. %setup -q %patch0 -p1 %patch1 -p1 +%patch2 -p1 +%patch3 -p1 %build %{__python} setup.py build @@ -114,6 +118,9 @@ cd tests && ./test_compose.sh %{_bindir}/%{name}-fedmsg-notification %changelog +* Tue Jan 24 2017 Dennis Gilmore - 4.1.12-4 +- add patches for pagure pr#517 + * Tue Jan 17 2017 Dennis Gilmore - 4.1.12-3 - add patch to replace ${basearch} in the ostree ref