add patches for pagure pr#517

Signed-off-by: Dennis Gilmore <dennis@ausil.us>

0001-image-build-Expand-arches-for-can_fail.patch

@@ -0,0 +1,104 @@
+From 6c708549c8f6632884bc55abed2f88afa1abe100 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
+Date: Mon, 23 Jan 2017 16:52:57 +0100
+Subject: [PATCH 1/2] image-build: Expand arches for can_fail
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+We need to work with a list of strings, not a comma-delimited single
+string.
+
+Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
+---
+ pungi/phases/image_build.py   |  2 +-
+ tests/test_imagebuildphase.py | 60 +++++++++++++++++++++++++++++++++++++++++++
+ 2 files changed, 61 insertions(+), 1 deletion(-)
+
+diff --git a/pungi/phases/image_build.py b/pungi/phases/image_build.py
+index 5cfdbb0..b795125 100644
+--- a/pungi/phases/image_build.py
++++ b/pungi/phases/image_build.py
+@@ -122,7 +122,7 @@ class ImageBuildPhase(base.PhaseLoggerMixin, base.ImageConfigMixin, base.ConfigG
+ 
+                 can_fail = image_conf['image-build'].pop('failable', [])
+                 if can_fail == ['*']:
+-                    can_fail = image_conf['image-build']['arches']
++                    can_fail = image_conf['image-build']['arches'].split(',')
+                 if can_fail:
+                     image_conf['image-build']['can_fail'] = ','.join(sorted(can_fail))
+ 
+diff --git a/tests/test_imagebuildphase.py b/tests/test_imagebuildphase.py
+index 86012ac..328b2b3 100644
+--- a/tests/test_imagebuildphase.py
++++ b/tests/test_imagebuildphase.py
+@@ -619,6 +619,66 @@ class TestImageBuildPhase(PungiTestCase):
+         self.assertItemsEqual(phase.pool.queue_put.mock_calls,
+                               [mock.call((compose, server_args))])
+ 
++    @mock.patch('pungi.phases.image_build.ThreadPool')
++    def test_failable_star(self, ThreadPool):
++        compose = DummyCompose(self.topdir, {
++            'image_build': {
++                '^Server$': [
++                    {
++                        'image-build': {
++                            'format': [('docker', 'tar.xz')],
++                            'name': 'Fedora-Docker-Base',
++                            'target': 'f24',
++                            'version': 'Rawhide',
++                            'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
++                            'kickstart': "fedora-docker-base.ks",
++                            'distro': 'Fedora-20',
++                            'disk_size': 3,
++                            'failable': ['*'],
++                        }
++                    }
++                ]
++            },
++            'koji_profile': 'koji',
++        })
++        compose.setup_optional()
++
++        self.assertValidConfig(compose.conf)
++
++        phase = ImageBuildPhase(compose)
++
++        phase.run()
++
++        # assert at least one thread was started
++        self.assertTrue(phase.pool.add.called)
++        server_args = {
++            "format": [('docker', 'tar.xz')],
++            "image_conf": {
++                'image-build': {
++                    'install_tree': self.topdir + '/compose/Server/$arch/os',
++                    'kickstart': 'fedora-docker-base.ks',
++                    'format': 'docker',
++                    'repo': self.topdir + '/compose/Server/$arch/os',
++                    'variant': compose.all_variants['Server'],
++                    'target': 'f24',
++                    'disk_size': 3,
++                    'name': 'Fedora-Docker-Base',
++                    'arches': 'amd64,x86_64',
++                    'version': 'Rawhide',
++                    'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
++                    'distro': 'Fedora-20',
++                    'can_fail': 'amd64,x86_64',
++                }
++            },
++            "conf_file": self.topdir + '/work/image-build/Server/docker_Fedora-Docker-Base.cfg',
++            "image_dir": self.topdir + '/compose/Server/%(arch)s/images',
++            "relative_image_dir": 'Server/%(arch)s/images',
++            "link_type": 'hardlink-or-copy',
++            "scratch": False,
++        }
++        self.assertItemsEqual(phase.pool.queue_put.mock_calls,
++                              [mock.call((compose, server_args))])
++
+ 
+ class TestCreateImageBuildThread(PungiTestCase):
+ 
+-- 
+2.11.0
+

0001-replace-basearch-when-updating-the-ref.patch

@@ -1,38 +1,66 @@
-From 1e9962be27718171e9291c1b62983229ca49b907 Mon Sep 17 00:00:00 2001
+From 59dd4dbcd84ce30faa6558ad0d052370077d3fe5 Mon Sep 17 00:00:00 2001
 From: Dennis Gilmore <dennis@ausil.us>
-Date: Wed, 18 Jan 2017 14:22:43 -0600
+Date: Jan 19 2017 03:21:26 +0000
-Subject: [PATCH] replace ${basearch} when updating the ref
+Subject: replace ${basearch} when updating the ref
+
 
 pungi-make-ostree has to run on the target arch so that rpm
-scriptlets can be ran.as a reult we can ask rpm what the
+scriptlets can be ran. as a reult we can ask rpm what the
-basearch is for the running environment
+basearch is for the running environment. For notifications
+we have to pass in the arch we are running for.
 
 Signed-off-by: Dennis Gilmore <dennis@ausil.us>
+
 ---
- pungi/ostree/tree.py | 3 +++
- 1 file changed, 3 insertions(+)
 
-diff --git a/pungi/ostree/tree.py b/pungi/ostree/tree.py
+diff --git a/pungi/ostree/utils.py b/pungi/ostree/utils.py
-index 68b6d70..94a0be2 100644
+index 02540c9..d80fb9f 100644
---- a/pungi/ostree/tree.py
+--- a/pungi/ostree/utils.py
-+++ b/pungi/ostree/tree.py
++++ b/pungi/ostree/utils.py
-@@ -16,6 +16,7 @@
+@@ -17,6 +17,7 @@
- 
+ import datetime
- import os
  import json
+ import os
 +import rpmUtils.arch
- from kobo import shortcuts
  
  from pungi.util import makedirs
-@@ -67,6 +68,8 @@ class Tree(OSTree):
+ 
-             return
+@@ -29,14 +30,21 @@ def make_log_file(log_dir, filename):
-         ref = get_ref_from_treefile(self.treefile)
+     return os.path.join(log_dir, '%s.log' % filename)
-         commitid = get_commitid_from_commitid_file(self.commitid_file)
+ 
-+        if ref is not None:
+ 
-+            ref = ref.replace('${basearch}', rpmUtils.arch.getBaseArch())
+-def get_ref_from_treefile(treefile):
-         print('Ref: %r, Commit ID: %r' % (ref, commitid))
+-    """Return ref name by parsing the tree config file"""
-         if ref and commitid:
++def get_ref_from_treefile(treefile, arch=None):
-             print('Updating ref')
++    """
--- 
++    Return ref name by parsing the tree config file. Replacing ${basearch} with
-2.11.0
++    the basearch of the architecture we are running on or of the passed in arch.
++    """
+     ref = None
+     if os.path.isfile(treefile):
+         with open(treefile, 'r') as f:
+             try:
+                 parsed = json.loads(f.read())
+-                ref = parsed['ref']
++                if arch is None:
++                    basearch = rpmUtils.arch.getBaseArch()
++                else:
++                    basearch = rpmUtils.arch.getBaseArch(arch)
++                ref = parsed['ref'].replace('${basearch}', basearch)
+             except Exception as e:
+                 print('Unable to get ref from treefile: %s' % e)
+     else:
+diff --git a/pungi/phases/ostree.py b/pungi/phases/ostree.py
+index ebe03d0..e61e969 100644
+--- a/pungi/phases/ostree.py
++++ b/pungi/phases/ostree.py
+@@ -104,7 +104,7 @@ class OSTreeThread(WorkerThread):
+                              extra_config_file=extra_config_file)
+ 
+         if compose.notifier:
+-            ref = get_ref_from_treefile(os.path.join(repodir, config['treefile']))
++            ref = get_ref_from_treefile(os.path.join(repodir, config['treefile']), arch)
+             # 'pungi-make-ostree tree' writes commitid to commitid.log in logdir
+             commitid = get_commitid_from_commitid_file(os.path.join(self.logdir, 'commitid.log'))
+             compose.notifier.send('ostree',
 

0002-image-build-Pass-arches-around-as-a-list.patch

@@ -0,0 +1,241 @@
+From 8418b68fb0e1253f688b97c91df9ace6a38ebd02 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
+Date: Tue, 24 Jan 2017 08:36:33 +0100
+Subject: [PATCH 2/2] image-build: Pass arches around as a list
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+Instead of joining the arches as a comma separated string and splitting
+it again later. Ultimately we do need the original format to pass to
+koji wrapper, but we can produce that value later.
+
+Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
+---
+ pungi/phases/image_build.py   | 16 ++++++++++------
+ tests/test_imagebuildphase.py | 41 +++++++++++++++++++++--------------------
+ 2 files changed, 31 insertions(+), 26 deletions(-)
+
+diff --git a/pungi/phases/image_build.py b/pungi/phases/image_build.py
+index b795125..bd18d3a 100644
+--- a/pungi/phases/image_build.py
++++ b/pungi/phases/image_build.py
+@@ -75,7 +75,7 @@ class ImageBuildPhase(base.PhaseLoggerMixin, base.ImageConfigMixin, base.ConfigG
+     def _get_arches(self, image_conf, arches):
+         if 'arches' in image_conf['image-build']:
+             arches = set(image_conf['image-build'].get('arches', [])) & arches
+-        return ','.join(sorted(arches))
++        return sorted(arches)
+ 
+     def _set_release(self, image_conf):
+         """If release is set explicitly to None, replace it with date and respin."""
+@@ -122,9 +122,9 @@ class ImageBuildPhase(base.PhaseLoggerMixin, base.ImageConfigMixin, base.ConfigG
+ 
+                 can_fail = image_conf['image-build'].pop('failable', [])
+                 if can_fail == ['*']:
+-                    can_fail = image_conf['image-build']['arches'].split(',')
++                    can_fail = image_conf['image-build']['arches']
+                 if can_fail:
+-                    image_conf['image-build']['can_fail'] = ','.join(sorted(can_fail))
++                    image_conf['image-build']['can_fail'] = sorted(can_fail)
+ 
+                 cmd = {
+                     "format": format,
+@@ -162,8 +162,7 @@ class CreateImageBuildThread(WorkerThread):
+             self.worker(num, compose, variant, subvariant, cmd)
+ 
+     def worker(self, num, compose, variant, subvariant, cmd):
+-        arches = cmd["image_conf"]["image-build"]['arches'].split(',')
+-        failable_arches = self.failable_arches.split(',')
++        arches = cmd["image_conf"]["image-build"]['arches']
+         dash_arches = '-'.join(arches)
+         log_file = compose.paths.log.log_file(
+             dash_arches,
+@@ -180,6 +179,11 @@ class CreateImageBuildThread(WorkerThread):
+         # writes conf file for koji image-build
+         self.pool.log_info("Writing image-build config for %s.%s into %s" % (
+             variant, dash_arches, cmd["conf_file"]))
++
++        # Join the arches into a single string. This is the value expected by
++        # koji config file.
++        cmd["image_conf"]["image-build"]['arches'] = ','.join(cmd["image_conf"]["image-build"]['arches'])
++
+         koji_cmd = koji_wrapper.get_image_build_cmd(cmd["image_conf"],
+                                                     conf_file_dest=cmd["conf_file"],
+                                                     scratch=cmd['scratch'])
+@@ -207,7 +211,7 @@ class CreateImageBuildThread(WorkerThread):
+                         image_infos.append({'path': path, 'suffix': suffix, 'type': format, 'arch': arch})
+                         break
+ 
+-        if len(image_infos) != len(cmd['format']) * (len(arches) - len(failable_arches)):
++        if len(image_infos) != len(cmd['format']) * (len(arches) - len(self.failable_arches)):
+             self.pool.log_error(
+                 "Error in koji task %s. Expected to find same amount of images "
+                 "as in suffixes attr in image-build (%s) for each arch (%s). Got '%s'." %
+diff --git a/tests/test_imagebuildphase.py b/tests/test_imagebuildphase.py
+index 328b2b3..d3185e2 100644
+--- a/tests/test_imagebuildphase.py
++++ b/tests/test_imagebuildphase.py
+@@ -66,11 +66,11 @@ class TestImageBuildPhase(PungiTestCase):
+                     'target': 'f24',
+                     'disk_size': 3,
+                     'name': 'Fedora-Docker-Base',
+-                    'arches': 'amd64',
++                    'arches': ['amd64'],
+                     'version': 'Rawhide',
+                     'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
+                     'distro': 'Fedora-20',
+-                    'can_fail': 'x86_64',
++                    'can_fail': ['x86_64'],
+                 }
+             },
+             "conf_file": self.topdir + '/work/image-build/Client/docker_Fedora-Docker-Base.cfg',
+@@ -91,11 +91,11 @@ class TestImageBuildPhase(PungiTestCase):
+                     'target': 'f24',
+                     'disk_size': 3,
+                     'name': 'Fedora-Docker-Base',
+-                    'arches': 'amd64,x86_64',
++                    'arches': ['amd64', 'x86_64'],
+                     'version': 'Rawhide',
+                     'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
+                     'distro': 'Fedora-20',
+-                    'can_fail': 'x86_64',
++                    'can_fail': ['x86_64'],
+                 }
+             },
+             "conf_file": self.topdir + '/work/image-build/Server/docker_Fedora-Docker-Base.cfg',
+@@ -151,7 +151,7 @@ class TestImageBuildPhase(PungiTestCase):
+                     'target': 'f24',
+                     'disk_size': 3,
+                     'name': 'Fedora-Docker-Base',
+-                    'arches': 'amd64,x86_64',
++                    'arches': ['amd64', 'x86_64'],
+                     'version': 'Rawhide',
+                     'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
+                     'distro': 'Fedora-20',
+@@ -207,7 +207,7 @@ class TestImageBuildPhase(PungiTestCase):
+                     'target': 'f24',
+                     'disk_size': 3,
+                     'name': 'Fedora-Docker-Base',
+-                    'arches': 'amd64,x86_64',
++                    'arches': ['amd64', 'x86_64'],
+                     'version': '25',
+                     'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
+                     'distro': 'Fedora-20',
+@@ -305,7 +305,7 @@ class TestImageBuildPhase(PungiTestCase):
+                     'target': 'f24',
+                     'disk_size': 3,
+                     'name': 'Fedora-Docker-Base',
+-                    'arches': 'x86_64',
++                    'arches': ['x86_64'],
+                     'version': 'Rawhide',
+                     'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
+                     'distro': 'Fedora-20',
+@@ -369,7 +369,7 @@ class TestImageBuildPhase(PungiTestCase):
+                     'target': 'f24',
+                     'disk_size': 3,
+                     'name': 'Fedora-Docker-Base',
+-                    'arches': 'x86_64',
++                    'arches': ['x86_64'],
+                     'version': 'Rawhide',
+                     'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
+                     'distro': 'Fedora-20',
+@@ -430,7 +430,7 @@ class TestImageBuildPhase(PungiTestCase):
+                     'target': 'f24',
+                     'disk_size': 3,
+                     'name': 'Fedora-Docker-Base',
+-                    'arches': 'x86_64',
++                    'arches': ['x86_64'],
+                     'version': 'Rawhide',
+                     'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
+                     'distro': 'Fedora-20',
+@@ -603,11 +603,11 @@ class TestImageBuildPhase(PungiTestCase):
+                     'target': 'f24',
+                     'disk_size': 3,
+                     'name': 'Fedora-Docker-Base',
+-                    'arches': 'x86_64',
++                    'arches': ['x86_64'],
+                     'version': 'Rawhide',
+                     'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
+                     'distro': 'Fedora-20',
+-                    'can_fail': 'x86_64',
++                    'can_fail': ['x86_64'],
+                 }
+             },
+             "conf_file": self.topdir + '/work/image-build/Server-optional/docker_Fedora-Docker-Base.cfg',
+@@ -663,11 +663,11 @@ class TestImageBuildPhase(PungiTestCase):
+                     'target': 'f24',
+                     'disk_size': 3,
+                     'name': 'Fedora-Docker-Base',
+-                    'arches': 'amd64,x86_64',
++                    'arches': ['amd64', 'x86_64'],
+                     'version': 'Rawhide',
+                     'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
+                     'distro': 'Fedora-20',
+-                    'can_fail': 'amd64,x86_64',
++                    'can_fail': ['amd64', 'x86_64'],
+                 }
+             },
+             "conf_file": self.topdir + '/work/image-build/Server/docker_Fedora-Docker-Base.cfg',
+@@ -703,7 +703,7 @@ class TestCreateImageBuildThread(PungiTestCase):
+                     'target': 'f24',
+                     'disk_size': 3,
+                     'name': 'Fedora-Docker-Base',
+-                    'arches': 'amd64,x86_64',
++                    'arches': ['amd64', 'x86_64'],
+                     'version': 'Rawhide',
+                     'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
+                     'distro': 'Fedora-20',
+@@ -743,6 +743,7 @@ class TestCreateImageBuildThread(PungiTestCase):
+         with mock.patch('time.sleep'):
+             t.process((compose, cmd), 1)
+ 
++        self.assertEqual(cmd['image_conf']['image-build']['arches'], 'amd64,x86_64')
+         self.assertItemsEqual(
+             koji_wrapper.get_image_build_cmd.call_args_list,
+             [mock.call(cmd['image_conf'],
+@@ -832,11 +833,11 @@ class TestCreateImageBuildThread(PungiTestCase):
+                     'target': 'f24',
+                     'disk_size': 3,
+                     'name': 'Fedora-Docker-Base',
+-                    'arches': 'amd64,x86_64',
++                    'arches': ['amd64', 'x86_64'],
+                     'version': 'Rawhide',
+                     'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
+                     'distro': 'Fedora-20',
+-                    "can_fail": 'amd64,x86_64',
++                    "can_fail": ['amd64', 'x86_64'],
+                 }
+             },
+             "conf_file": 'amd64,x86_64-Client-Fedora-Docker-Base-docker',
+@@ -880,11 +881,11 @@ class TestCreateImageBuildThread(PungiTestCase):
+                     'target': 'f24',
+                     'disk_size': 3,
+                     'name': 'Fedora-Docker-Base',
+-                    'arches': 'amd64,x86_64',
++                    'arches': ['amd64', 'x86_64'],
+                     'version': 'Rawhide',
+                     'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
+                     'distro': 'Fedora-20',
+-                    'can_fail': 'amd64,x86_64',
++                    'can_fail': ['amd64', 'x86_64'],
+                 }
+             },
+             "conf_file": 'amd64,x86_64-Client-Fedora-Docker-Base-docker',
+@@ -923,11 +924,11 @@ class TestCreateImageBuildThread(PungiTestCase):
+                     'target': 'f24',
+                     'disk_size': 3,
+                     'name': 'Fedora-Docker-Base',
+-                    'arches': 'amd64,x86_64',
++                    'arches': ['amd64', 'x86_64'],
+                     'version': 'Rawhide',
+                     'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
+                     'distro': 'Fedora-20',
+-                    'can_fail': 'amd64',
++                    'can_fail': ['amd64'],
+                 }
+             },
+             "conf_file": 'amd64,x86_64-Client-Fedora-Docker-Base-docker',
+-- 
+2.11.0
+

pungi.spec

@@ -1,6 +1,6 @@
 Name:           pungi
 Version:        4.1.12
-Release:        3%{?dist}
+Release:        4%{?dist}
 Summary:        Distribution compose tool
 
 Group:          Development/Tools
@@ -10,6 +10,8 @@ Source0:        https://pagure.io/releases/%{name}/%{name}-%{version}.tar.bz2
 # from https://pagure.io/fork/puiterwijk/pungi/c/63aa7db5aad4690b72898d01fa82270c1f503241
 Patch0:         63aa7db5aad4690b72898d01fa82270c1f503241.patch
 Patch1:         0001-replace-basearch-when-updating-the-ref.patch
+Patch2:         0001-image-build-Expand-arches-for-can_fail.patch
+Patch3:         0002-image-build-Pass-arches-around-as-a-list.patch
 
 BuildRequires:  python-nose, python-mock
 BuildRequires:  python-devel, python-setuptools, python2-productmd >= 1.3
@@ -73,6 +75,8 @@ notification to Fedora Message Bus.
 %setup -q
 %patch0 -p1 
 %patch1 -p1
+%patch2 -p1
+%patch3 -p1
 
 %build
 %{__python} setup.py build
@@ -114,6 +118,9 @@ cd tests && ./test_compose.sh
 %{_bindir}/%{name}-fedmsg-notification
 
 %changelog
+* Tue Jan 24 2017 Dennis Gilmore <dennis@ausil.us> - 4.1.12-4
+- add patches for pagure pr#517
+
 * Tue Jan 17 2017 Dennis Gilmore <dennis@ausil.us> - 4.1.12-3
 - add patch to replace ${basearch} in the ostree ref