add patch to print more info for missing rpms

- add patch to clone repo with extra lorax templates for ostree_installer

Signed-off-by: Dennis Gilmore <dennis@ausil.us>

0001-ostree-installer-Clone-repo-with-templates.patch

@@ -0,0 +1,260 @@
+From a696eb4f3ada1f6872dc6967f61bf8af905ab107 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
+Date: Mon, 11 Apr 2016 10:25:10 +0200
+Subject: [PATCH 1/2] [ostree-installer] Clone repo with templates
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+We can't assume the templates will just be available. This patch adds a
+configuration option to point to the git repo with them. It will be
+cloned at compose box and relative paths to templates will be made
+absolute respective to this clone.
+
+Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
+---
+ doc/configuration.rst                |  13 +++-
+ pungi/phases/ostree_installer.py     |  30 ++++++++-
+ tests/test_ostree_installer_phase.py | 117 +++++++++++++++++++++++++++++++++++
+ 3 files changed, 155 insertions(+), 5 deletions(-)
+
+diff --git a/doc/configuration.rst b/doc/configuration.rst
+index ff26b44..94704ae 100644
+--- a/doc/configuration.rst
++++ b/doc/configuration.rst
+@@ -1005,6 +1005,13 @@ an OSTree repository. This always runs in Koji as a ``runroot`` task.
+     * ``add_arch_template`` -- (*[str]*)
+     * ``add_template_var`` -- (*[str]*)
+     * ``add_arch_template_var`` -- (*[str]*)
++    * ``template_repo`` -- (*str*) Git repository with extra templates.
++    * ``template_branch`` -- (*str*) Branch to use from ``template_repo``.
++
++    The templates can either be absolute paths, in which case they will be used
++    as configured; or they can be relative paths, in which case
++    ``template_repo`` needs to point to a Git repository from which to take the
++    templates.
+ 
+ 
+ Example config
+@@ -1018,17 +1025,19 @@ Example config
+                 "release": None,
+                 "filename": "%(release_short)s-%(variant)s-%(arch)s-%(version)s-%(compose_date)s.iso",
+                 "installpkgs": ["fedora-productimg-atomic"],
+-                "add_template": ["/spin-kickstarts/atomic-installer/lorax-configure-repo.tmpl"],
++                "add_template": ["atomic-installer/lorax-configure-repo.tmpl"],
+                 "add_template_var": [
+                     "ostree_osname=fedora-atomic",
+                     "ostree_ref=fedora-atomic/Rawhide/x86_64/docker-host",
+                 ],
+-                "add_arch_template": ["/spin-kickstarts/atomic-installer/lorax-embed-repo.tmpl"],
++                "add_arch_template": ["atomic-installer/lorax-embed-repo.tmpl"],
+                 "add_arch_template_var": [
+                     "ostree_repo=https://kojipkgs.fedoraproject.org/compose/atomic/Rawhide/",
+                     "ostree_osname=fedora-atomic",
+                     "ostree_ref=fedora-atomic/Rawhide/x86_64/docker-host",
+                 ]
++                'template_repo': 'https://git.fedorahosted.org/git/spin-kickstarts.git',
++                'template_branch': 'f24',
+             }
+         })
+     ]
+diff --git a/pungi/phases/ostree_installer.py b/pungi/phases/ostree_installer.py
+index c73bbea..7399874 100644
+--- a/pungi/phases/ostree_installer.py
++++ b/pungi/phases/ostree_installer.py
+@@ -10,7 +10,7 @@ from kobo import shortcuts
+ from .base import ConfigGuardedPhase
+ from .. import util
+ from ..paths import translate_path
+-from ..wrappers import kojiwrapper, iso, lorax
++from ..wrappers import kojiwrapper, iso, lorax, scm
+ 
+ 
+ class OstreeInstallerPhase(ConfigGuardedPhase):
+@@ -56,6 +56,9 @@ class OstreeInstallerThread(WorkerThread):
+         output_dir = os.path.join(compose.paths.work.topdir(arch), variant.uid, 'ostree_installer')
+         util.makedirs(os.path.dirname(output_dir))
+ 
++        self.template_dir = os.path.join(compose.paths.work.topdir(arch), variant.uid, 'lorax_templates')
++        self._clone_templates(config.get('template_repo'), config.get('template_branch'))
++
+         self._run_ostree_cmd(compose, variant, arch, config, source_repo, output_dir)
+ 
+         disc_type = compose.conf.get('disc_types', {}).get('dvd', 'dvd')
+@@ -65,6 +68,13 @@ class OstreeInstallerThread(WorkerThread):
+         self._add_to_manifest(compose, variant, arch, filename)
+         self.pool.log_info('[DONE ] %s' % msg)
+ 
++    def _clone_templates(self, url, branch='master'):
++        if not url:
++            self.template_dir = None
++            return
++        scm.get_dir_from_scm({'scm': 'git', 'repo': url, 'branch': branch, 'dir': '.'},
++                             self.template_dir, logger=self.pool._logger)
++
+     def _get_release(self, compose, config):
+         if 'release' in config and config['release'] is None:
+             return compose.image_release
+@@ -105,6 +115,20 @@ class OstreeInstallerThread(WorkerThread):
+             pass
+         compose.im.add(variant.uid, arch, img)
+ 
++    def _get_templates(self, config, key):
++        """Retrieve all templates from configuration and make sure the paths
++        are absolute. Raises RuntimeError if template repo is needed but not
++        configured.
++        """
++        templates = []
++        for template in config.get(key, []):
++            if template[0] != '/':
++                if not self.template_dir:
++                    raise RuntimeError('Relative path to template without setting template_repo.')
++                template = os.path.join(self.template_dir, template)
++            templates.append(template)
++        return templates
++
+     def _run_ostree_cmd(self, compose, variant, arch, config, source_repo, output_dir):
+         lorax_wrapper = lorax.LoraxWrapper()
+         cmd = lorax_wrapper.get_lorax_cmd(
+@@ -116,8 +140,8 @@ class OstreeInstallerThread(WorkerThread):
+             variant=variant.uid,
+             nomacboot=True,
+             buildinstallpackages=config.get('installpkgs'),
+-            add_template=config.get('add_template'),
+-            add_arch_template=config.get('add_arch_template'),
++            add_template=self._get_templates(config, 'add_template'),
++            add_arch_template=self._get_templates(config, 'add_arch_template'),
+             add_template_var=config.get('add_template_var'),
+             add_arch_template_var=config.get('add_arch_template_var')
+         )
+diff --git a/tests/test_ostree_installer_phase.py b/tests/test_ostree_installer_phase.py
+index 6a8205c..7007a3f 100644
+--- a/tests/test_ostree_installer_phase.py
++++ b/tests/test_ostree_installer_phase.py
+@@ -172,6 +172,123 @@ class OstreeThreadTest(helpers.PungiTestCase):
+     @mock.patch('pungi.wrappers.iso.IsoWrapper')
+     @mock.patch('os.link')
+     @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
++    def test_fail_with_relative_template_path_but_no_repo(self, KojiWrapper, link,
++                                                          IsoWrapper, get_file_size,
++                                                          get_mtime, ImageCls, run):
++        compose = helpers.DummyCompose(self.topdir, {
++            'release_name': 'Fedora',
++            'release_version': 'Rawhide',
++            'koji_profile': 'koji',
++            'runroot_tag': 'rrt',
++        })
++        pool = mock.Mock()
++        cfg = {
++            'source_repo_from': 'Everything',
++            'release': '20160321.n.0',
++            'filename': 'Fedora-Atomic.iso',
++            'add_template': ['some-file.txt'],
++        }
++        koji = KojiWrapper.return_value
++        koji.run_runroot_cmd.return_value = {
++            'task_id': 1234,
++            'retcode': 0,
++            'output': 'Foo bar\n',
++        }
++        get_file_size.return_value = 1024
++        get_mtime.return_value = 13579
++
++        t = ostree.OstreeInstallerThread(pool)
++
++        with self.assertRaises(RuntimeError) as ctx:
++            t.process((compose, compose.variants['Everything'], 'x86_64', cfg), 1)
++
++        self.assertIn('template_repo', str(ctx.exception))
++
++    @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
++    @mock.patch('kobo.shortcuts.run')
++    @mock.patch('productmd.images.Image')
++    @mock.patch('pungi.util.get_mtime')
++    @mock.patch('pungi.util.get_file_size')
++    @mock.patch('pungi.wrappers.iso.IsoWrapper')
++    @mock.patch('os.link')
++    @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
++    def test_run_clone_templates(self, KojiWrapper, link, IsoWrapper,
++                                 get_file_size, get_mtime, ImageCls, run,
++                                 get_dir_from_scm):
++        compose = helpers.DummyCompose(self.topdir, {
++            'release_name': 'Fedora',
++            'release_version': 'Rawhide',
++            'koji_profile': 'koji',
++            'runroot_tag': 'rrt',
++        })
++        pool = mock.Mock()
++        cfg = {
++            'source_repo_from': 'Everything',
++            'release': '20160321.n.0',
++            'filename': 'Fedora-Atomic.iso',
++            'add_template': ['some_file.txt'],
++            'add_arch_template': ['other_file.txt'],
++            'template_repo': 'git://example.com/templates.git',
++            'template_branch': 'f24',
++        }
++        koji = KojiWrapper.return_value
++        koji.run_runroot_cmd.return_value = {
++            'task_id': 1234,
++            'retcode': 0,
++            'output': 'Foo bar\n',
++        }
++        get_file_size.return_value = 1024
++        get_mtime.return_value = 13579
++        final_iso_path = self.topdir + '/compose/Everything/x86_64/iso/image-name'
++        templ_dir = self.topdir + '/work/x86_64/Everything/lorax_templates'
++
++        t = ostree.OstreeInstallerThread(pool)
++
++        t.process((compose, compose.variants['Everything'], 'x86_64', cfg), 1)
++
++        self.assertEqual(get_dir_from_scm.call_args_list,
++                         [mock.call({'scm': 'git', 'repo': 'git://example.com/templates.git',
++                                     'branch': 'f24', 'dir': '.'},
++                                    templ_dir, logger=pool._logger)])
++        self.assertEqual(koji.get_runroot_cmd.call_args_list,
++                         [mock.call('rrt', 'x86_64',
++                                    ['lorax',
++                                     '--product=Fedora',
++                                     '--version=Rawhide',
++                                     '--release=20160321.n.0',
++                                     '--source=file://{}/compose/Everything/x86_64/os'.format(self.topdir),
++                                     '--variant=Everything',
++                                     '--nomacboot',
++                                     '--add-template={}/some_file.txt'.format(templ_dir),
++                                     '--add-arch-template={}/other_file.txt'.format(templ_dir),
++                                     self.topdir + '/work/x86_64/Everything/ostree_installer'],
++                                    channel=None, mounts=[self.topdir],
++                                    packages=['pungi', 'lorax'],
++                                    task_id=True, use_shell=True)])
++        self.assertEqual(koji.run_runroot_cmd.call_args_list,
++                         [mock.call(koji.get_runroot_cmd.return_value,
++                                    log_file=self.topdir + '/logs/x86_64/ostree_installer/runroot.log')])
++        self.assertEqual(link.call_args_list,
++                         [mock.call(self.topdir + '/work/x86_64/Everything/ostree_installer/images/boot.iso',
++                                    final_iso_path)])
++        self.assertEqual(get_file_size.call_args_list, [mock.call(final_iso_path)])
++        self.assertEqual(get_mtime.call_args_list, [mock.call(final_iso_path)])
++        self.assertImageAdded(compose, ImageCls, IsoWrapper)
++        self.assertEqual(compose.get_image_name.call_args_list,
++                         [mock.call('x86_64', compose.variants['Everything'],
++                                    disc_type='dvd', format='Fedora-Atomic.iso')])
++        self.assertTrue(os.path.isdir(self.topdir + '/work/x86_64/Everything/'))
++        self.assertFalse(os.path.isdir(self.topdir + '/work/x86_64/Everything/ostree_installer'))
++        self.assertEqual(run.call_args_list,
++                         [mock.call('cp -av {0}/work/x86_64/Everything/ostree_installer/* {0}/compose/Everything/x86_64/iso/'.format(self.topdir))])
++
++    @mock.patch('kobo.shortcuts.run')
++    @mock.patch('productmd.images.Image')
++    @mock.patch('pungi.util.get_mtime')
++    @mock.patch('pungi.util.get_file_size')
++    @mock.patch('pungi.wrappers.iso.IsoWrapper')
++    @mock.patch('os.link')
++    @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
+     def test_run_with_implicit_release(self, KojiWrapper, link, IsoWrapper,
+                                        get_file_size, get_mtime, ImageCls, run):
+         compose = helpers.DummyCompose(self.topdir, {
+-- 
+2.7.3
+

0002-pkgset-Print-more-detailed-logs-when-rpm-is-not-foun.patch

@@ -0,0 +1,56 @@
+From 31d922692f20e67a08e51b0cb8e6840e1db547a1 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
+Date: Mon, 11 Apr 2016 15:34:26 +0200
+Subject: [PATCH 2/2] [pkgset] Print more detailed logs when rpm is not found
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+Fixes: #265
+Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
+---
+ pungi/phases/pkgset/pkgsets.py | 9 +++++++--
+ 1 file changed, 7 insertions(+), 2 deletions(-)
+
+diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py
+index 834fe31..adc2d07 100644
+--- a/pungi/phases/pkgset/pkgsets.py
++++ b/pungi/phases/pkgset/pkgsets.py
+@@ -225,12 +225,14 @@ class KojiPackageSet(PackageSetBase):
+         rpm_path = None
+         found = False
+         pathinfo = self.koji_wrapper.koji_module.pathinfo
++        paths = []
+         for sigkey in self.sigkey_ordering:
+             if sigkey is None:
+                 # we're looking for *signed* copies here
+                 continue
+             sigkey = sigkey.lower()
+             rpm_path = os.path.join(pathinfo.build(build_info), pathinfo.signed(rpm_info, sigkey))
++            paths.append(rpm_path)
+             if os.path.isfile(rpm_path):
+                 found = True
+                 break
+@@ -239,14 +241,17 @@ class KojiPackageSet(PackageSetBase):
+             if None in self.sigkey_ordering:
+                 # use an unsigned copy (if allowed)
+                 rpm_path = os.path.join(pathinfo.build(build_info), pathinfo.rpm(rpm_info))
++                paths.append(rpm_path)
+                 if os.path.isfile(rpm_path):
+                     found = True
+             else:
+                 # or raise an exception
+-                raise RuntimeError("RPM %s not found for sigs: %s" % (rpm_info, self.sigkey_ordering))
++                raise RuntimeError("RPM %s not found for sigs: %s. Paths checked: %s"
++                                   % (rpm_info, self.sigkey_ordering, paths))
+ 
+         if not found:
+-            raise RuntimeError("Package not found: %s" % rpm_info)
++            raise RuntimeError("Package not found: %s. Paths checked: %s"
++                               % (rpm_info, paths))
+         return rpm_path
+ 
+     def populate(self, tag, event=None, inherit=True):
+-- 
+2.7.3
+

pungi.spec

@@ -1,12 +1,14 @@
 Name:           pungi
 Version:        4.1.3
-Release:        1%{?dist}
+Release:        2%{?dist}
 Summary:        Distribution compose tool
 
 Group:          Development/Tools
 License:        GPLv2
 URL:            https://pagure.io/pungi
 Source0:        https://fedorahosted.org/pungi/attachment/wiki/%{version}/%{name}-%{version}.tar.bz2
+Patch0:         0001-ostree-installer-Clone-repo-with-templates.patch
+Patch1:         0002-pkgset-Print-more-detailed-logs-when-rpm-is-not-foun.patch
 
 BuildRequires:  python-nose, python-nose-cov, python-mock
 BuildRequires:  python-devel, python-setuptools, python2-productmd
@@ -48,6 +50,8 @@ A tool to create anaconda based installation trees/isos of a set of rpms.
 
 %prep
 %setup -q
+%patch0 -p1
+%patch1 -p1
 
 %build
 %{__python} setup.py build
@@ -83,6 +87,10 @@ cd tests && ./test_compose.sh
 /var/cache/pungi
 
 %changelog
+* Mon Apr 11 2016 Dennis Gilmore <dennis@ausil.us> - 4.1.3-2
+- add patch to print more info for missing rpms
+- add patch to clone repo with extra lorax templates for ostree_installer
+
 * Fri Apr 08 2016 Dennis Gilmore <dennis@ausil.us> - 4.1.3-1
 - enable the compose test (dennis)
 - [ostree-installer] Copy all lorax outputs (lsedlar)