almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

add patch to print more info for missing rpms

Browse Source 
- add patch to clone repo with extra lorax templates for ostree_installer

Signed-off-by: Dennis Gilmore <dennis@ausil.us>
This commit is contained in:
Dennis Gilmore 2016-04-11 12:27:19 -05:00
parent 684d9d06eb
commit 16b5af4640
3 changed files with 325 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

260
0001-ostree-installer-Clone-repo-with-templates.patch Normal file
View File

@ -0,0 +1,260 @@
From a696eb4f3ada1f6872dc6967f61bf8af905ab107 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
Date: Mon, 11 Apr 2016 10:25:10 +0200
Subject: [PATCH 1/2] [ostree-installer] Clone repo with templates
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
We can't assume the templates will just be available. This patch adds a
configuration option to point to the git repo with them. It will be
cloned at compose box and relative paths to templates will be made
absolute respective to this clone.
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
---
doc/configuration.rst | 13 +++-
pungi/phases/ostree_installer.py | 30 ++++++++-
tests/test_ostree_installer_phase.py | 117 +++++++++++++++++++++++++++++++++++
3 files changed, 155 insertions(+), 5 deletions(-)
diff --git a/doc/configuration.rst b/doc/configuration.rst
index ff26b44..94704ae 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -1005,6 +1005,13 @@ an OSTree repository. This always runs in Koji as a ``runroot`` task.
* ``add_arch_template`` -- (*[str]*)
* ``add_template_var`` -- (*[str]*)
* ``add_arch_template_var`` -- (*[str]*)
+ * ``template_repo`` -- (*str*) Git repository with extra templates.
+ * ``template_branch`` -- (*str*) Branch to use from ``template_repo``.
+
+ The templates can either be absolute paths, in which case they will be used
+ as configured; or they can be relative paths, in which case
+ ``template_repo`` needs to point to a Git repository from which to take the
+ templates.
Example config
@@ -1018,17 +1025,19 @@ Example config
"release": None,
"filename": "%(release_short)s-%(variant)s-%(arch)s-%(version)s-%(compose_date)s.iso",
"installpkgs": ["fedora-productimg-atomic"],
- "add_template": ["/spin-kickstarts/atomic-installer/lorax-configure-repo.tmpl"],
+ "add_template": ["atomic-installer/lorax-configure-repo.tmpl"],
"add_template_var": [
"ostree_osname=fedora-atomic",
"ostree_ref=fedora-atomic/Rawhide/x86_64/docker-host",
],
- "add_arch_template": ["/spin-kickstarts/atomic-installer/lorax-embed-repo.tmpl"],
+ "add_arch_template": ["atomic-installer/lorax-embed-repo.tmpl"],
"add_arch_template_var": [
"ostree_repo=https://kojipkgs.fedoraproject.org/compose/atomic/Rawhide/",
"ostree_osname=fedora-atomic",
"ostree_ref=fedora-atomic/Rawhide/x86_64/docker-host",
]
+ 'template_repo': 'https://git.fedorahosted.org/git/spin-kickstarts.git',
+ 'template_branch': 'f24',
}
})
]
diff --git a/pungi/phases/ostree_installer.py b/pungi/phases/ostree_installer.py
index c73bbea..7399874 100644
--- a/pungi/phases/ostree_installer.py
+++ b/pungi/phases/ostree_installer.py
@@ -10,7 +10,7 @@ from kobo import shortcuts
from .base import ConfigGuardedPhase
from .. import util
from ..paths import translate_path
-from ..wrappers import kojiwrapper, iso, lorax
+from ..wrappers import kojiwrapper, iso, lorax, scm
class OstreeInstallerPhase(ConfigGuardedPhase):
@@ -56,6 +56,9 @@ class OstreeInstallerThread(WorkerThread):
output_dir = os.path.join(compose.paths.work.topdir(arch), variant.uid, 'ostree_installer')
util.makedirs(os.path.dirname(output_dir))
+ self.template_dir = os.path.join(compose.paths.work.topdir(arch), variant.uid, 'lorax_templates')
+ self._clone_templates(config.get('template_repo'), config.get('template_branch'))
+
self._run_ostree_cmd(compose, variant, arch, config, source_repo, output_dir)
disc_type = compose.conf.get('disc_types', {}).get('dvd', 'dvd')
@@ -65,6 +68,13 @@ class OstreeInstallerThread(WorkerThread):
self._add_to_manifest(compose, variant, arch, filename)
self.pool.log_info('[DONE ] %s' % msg)
+ def _clone_templates(self, url, branch='master'):
+ if not url:
+ self.template_dir = None
+ return
+ scm.get_dir_from_scm({'scm': 'git', 'repo': url, 'branch': branch, 'dir': '.'},
+ self.template_dir, logger=self.pool._logger)
+
def _get_release(self, compose, config):
if 'release' in config and config['release'] is None:
return compose.image_release
@@ -105,6 +115,20 @@ class OstreeInstallerThread(WorkerThread):
pass
compose.im.add(variant.uid, arch, img)
+ def _get_templates(self, config, key):
+ """Retrieve all templates from configuration and make sure the paths
+ are absolute. Raises RuntimeError if template repo is needed but not
+ configured.
+ """
+ templates = []
+ for template in config.get(key, []):
+ if template[0] != '/':
+ if not self.template_dir:
+ raise RuntimeError('Relative path to template without setting template_repo.')
+ template = os.path.join(self.template_dir, template)
+ templates.append(template)
+ return templates
+
def _run_ostree_cmd(self, compose, variant, arch, config, source_repo, output_dir):
lorax_wrapper = lorax.LoraxWrapper()
cmd = lorax_wrapper.get_lorax_cmd(
@@ -116,8 +140,8 @@ class OstreeInstallerThread(WorkerThread):
variant=variant.uid,
nomacboot=True,
buildinstallpackages=config.get('installpkgs'),
- add_template=config.get('add_template'),
- add_arch_template=config.get('add_arch_template'),
+ add_template=self._get_templates(config, 'add_template'),
+ add_arch_template=self._get_templates(config, 'add_arch_template'),
add_template_var=config.get('add_template_var'),
add_arch_template_var=config.get('add_arch_template_var')
)
diff --git a/tests/test_ostree_installer_phase.py b/tests/test_ostree_installer_phase.py
index 6a8205c..7007a3f 100644
--- a/tests/test_ostree_installer_phase.py
+++ b/tests/test_ostree_installer_phase.py
@@ -172,6 +172,123 @@ class OstreeThreadTest(helpers.PungiTestCase):
@mock.patch('pungi.wrappers.iso.IsoWrapper')
@mock.patch('os.link')
@mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
+ def test_fail_with_relative_template_path_but_no_repo(self, KojiWrapper, link,
+ IsoWrapper, get_file_size,
+ get_mtime, ImageCls, run):
+ compose = helpers.DummyCompose(self.topdir, {
+ 'release_name': 'Fedora',
+ 'release_version': 'Rawhide',
+ 'koji_profile': 'koji',
+ 'runroot_tag': 'rrt',
+ })
+ pool = mock.Mock()
+ cfg = {
+ 'source_repo_from': 'Everything',
+ 'release': '20160321.n.0',
+ 'filename': 'Fedora-Atomic.iso',
+ 'add_template': ['some-file.txt'],
+ }
+ koji = KojiWrapper.return_value
+ koji.run_runroot_cmd.return_value = {
+ 'task_id': 1234,
+ 'retcode': 0,
+ 'output': 'Foo bar\n',
+ }
+ get_file_size.return_value = 1024
+ get_mtime.return_value = 13579
+
+ t = ostree.OstreeInstallerThread(pool)
+
+ with self.assertRaises(RuntimeError) as ctx:
+ t.process((compose, compose.variants['Everything'], 'x86_64', cfg), 1)
+
+ self.assertIn('template_repo', str(ctx.exception))
+
+ @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
+ @mock.patch('kobo.shortcuts.run')
+ @mock.patch('productmd.images.Image')
+ @mock.patch('pungi.util.get_mtime')
+ @mock.patch('pungi.util.get_file_size')
+ @mock.patch('pungi.wrappers.iso.IsoWrapper')
+ @mock.patch('os.link')
+ @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
+ def test_run_clone_templates(self, KojiWrapper, link, IsoWrapper,
+ get_file_size, get_mtime, ImageCls, run,
+ get_dir_from_scm):
+ compose = helpers.DummyCompose(self.topdir, {
+ 'release_name': 'Fedora',
+ 'release_version': 'Rawhide',
+ 'koji_profile': 'koji',
+ 'runroot_tag': 'rrt',
+ })
+ pool = mock.Mock()
+ cfg = {
+ 'source_repo_from': 'Everything',
+ 'release': '20160321.n.0',
+ 'filename': 'Fedora-Atomic.iso',
+ 'add_template': ['some_file.txt'],
+ 'add_arch_template': ['other_file.txt'],
+ 'template_repo': 'git://example.com/templates.git',
+ 'template_branch': 'f24',
+ }
+ koji = KojiWrapper.return_value
+ koji.run_runroot_cmd.return_value = {
+ 'task_id': 1234,
+ 'retcode': 0,
+ 'output': 'Foo bar\n',
+ }
+ get_file_size.return_value = 1024
+ get_mtime.return_value = 13579
+ final_iso_path = self.topdir + '/compose/Everything/x86_64/iso/image-name'
+ templ_dir = self.topdir + '/work/x86_64/Everything/lorax_templates'
+
+ t = ostree.OstreeInstallerThread(pool)
+
+ t.process((compose, compose.variants['Everything'], 'x86_64', cfg), 1)
+
+ self.assertEqual(get_dir_from_scm.call_args_list,
+ [mock.call({'scm': 'git', 'repo': 'git://example.com/templates.git',
+ 'branch': 'f24', 'dir': '.'},
+ templ_dir, logger=pool._logger)])
+ self.assertEqual(koji.get_runroot_cmd.call_args_list,
+ [mock.call('rrt', 'x86_64',
+ ['lorax',
+ '--product=Fedora',
+ '--version=Rawhide',
+ '--release=20160321.n.0',
+ '--source=file://{}/compose/Everything/x86_64/os'.format(self.topdir),
+ '--variant=Everything',
+ '--nomacboot',
+ '--add-template={}/some_file.txt'.format(templ_dir),
+ '--add-arch-template={}/other_file.txt'.format(templ_dir),
+ self.topdir + '/work/x86_64/Everything/ostree_installer'],
+ channel=None, mounts=[self.topdir],
+ packages=['pungi', 'lorax'],
+ task_id=True, use_shell=True)])
+ self.assertEqual(koji.run_runroot_cmd.call_args_list,
+ [mock.call(koji.get_runroot_cmd.return_value,
+ log_file=self.topdir + '/logs/x86_64/ostree_installer/runroot.log')])
+ self.assertEqual(link.call_args_list,
+ [mock.call(self.topdir + '/work/x86_64/Everything/ostree_installer/images/boot.iso',
+ final_iso_path)])
+ self.assertEqual(get_file_size.call_args_list, [mock.call(final_iso_path)])
+ self.assertEqual(get_mtime.call_args_list, [mock.call(final_iso_path)])
+ self.assertImageAdded(compose, ImageCls, IsoWrapper)
+ self.assertEqual(compose.get_image_name.call_args_list,
+ [mock.call('x86_64', compose.variants['Everything'],
+ disc_type='dvd', format='Fedora-Atomic.iso')])
+ self.assertTrue(os.path.isdir(self.topdir + '/work/x86_64/Everything/'))
+ self.assertFalse(os.path.isdir(self.topdir + '/work/x86_64/Everything/ostree_installer'))
+ self.assertEqual(run.call_args_list,
+ [mock.call('cp -av {0}/work/x86_64/Everything/ostree_installer/* {0}/compose/Everything/x86_64/iso/'.format(self.topdir))])
+
+ @mock.patch('kobo.shortcuts.run')
+ @mock.patch('productmd.images.Image')
+ @mock.patch('pungi.util.get_mtime')
+ @mock.patch('pungi.util.get_file_size')
+ @mock.patch('pungi.wrappers.iso.IsoWrapper')
+ @mock.patch('os.link')
+ @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
def test_run_with_implicit_release(self, KojiWrapper, link, IsoWrapper,
get_file_size, get_mtime, ImageCls, run):
compose = helpers.DummyCompose(self.topdir, {
--
2.7.3

56
0002-pkgset-Print-more-detailed-logs-when-rpm-is-not-foun.patch Normal file
View File

@ -0,0 +1,56 @@
From 31d922692f20e67a08e51b0cb8e6840e1db547a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
Date: Mon, 11 Apr 2016 15:34:26 +0200
Subject: [PATCH 2/2] [pkgset] Print more detailed logs when rpm is not found
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Fixes: #265
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
---
pungi/phases/pkgset/pkgsets.py | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py
index 834fe31..adc2d07 100644
--- a/pungi/phases/pkgset/pkgsets.py
+++ b/pungi/phases/pkgset/pkgsets.py
@@ -225,12 +225,14 @@ class KojiPackageSet(PackageSetBase):
rpm_path = None
found = False
pathinfo = self.koji_wrapper.koji_module.pathinfo
+ paths = []
for sigkey in self.sigkey_ordering:
if sigkey is None:
# we're looking for *signed* copies here
continue
sigkey = sigkey.lower()
rpm_path = os.path.join(pathinfo.build(build_info), pathinfo.signed(rpm_info, sigkey))
+ paths.append(rpm_path)
if os.path.isfile(rpm_path):
found = True
break
@@ -239,14 +241,17 @@ class KojiPackageSet(PackageSetBase):
if None in self.sigkey_ordering:
# use an unsigned copy (if allowed)
rpm_path = os.path.join(pathinfo.build(build_info), pathinfo.rpm(rpm_info))
+ paths.append(rpm_path)
if os.path.isfile(rpm_path):
found = True
else:
# or raise an exception
- raise RuntimeError("RPM %s not found for sigs: %s" % (rpm_info, self.sigkey_ordering))
+ raise RuntimeError("RPM %s not found for sigs: %s. Paths checked: %s"
+ % (rpm_info, self.sigkey_ordering, paths))
if not found:
- raise RuntimeError("Package not found: %s" % rpm_info)
+ raise RuntimeError("Package not found: %s. Paths checked: %s"
+ % (rpm_info, paths))
return rpm_path
def populate(self, tag, event=None, inherit=True):
--
2.7.3

10
pungi.spec
View File

@ -1,12 +1,14 @@
Name: pungi Name: pungi
Version: 4.1.3 Version: 4.1.3
Release: 1%{?dist} Release: 2%{?dist}
Summary: Distribution compose tool Summary: Distribution compose tool
Group: Development/Tools Group: Development/Tools
License: GPLv2 License: GPLv2
URL: https://pagure.io/pungi URL: https://pagure.io/pungi
Source0: https://fedorahosted.org/pungi/attachment/wiki/%{version}/%{name}-%{version}.tar.bz2 Source0: https://fedorahosted.org/pungi/attachment/wiki/%{version}/%{name}-%{version}.tar.bz2
Patch0: 0001-ostree-installer-Clone-repo-with-templates.patch
Patch1: 0002-pkgset-Print-more-detailed-logs-when-rpm-is-not-foun.patch
BuildRequires: python-nose, python-nose-cov, python-mock BuildRequires: python-nose, python-nose-cov, python-mock
BuildRequires: python-devel, python-setuptools, python2-productmd BuildRequires: python-devel, python-setuptools, python2-productmd
@ -48,6 +50,8 @@ A tool to create anaconda based installation trees/isos of a set of rpms.
%prep %prep
%setup -q %setup -q
%patch0 -p1
%patch1 -p1
%build %build
%{__python} setup.py build %{__python} setup.py build
@ -83,6 +87,10 @@ cd tests && ./test_compose.sh
/var/cache/pungi /var/cache/pungi
%changelog %changelog
* Mon Apr 11 2016 Dennis Gilmore <dennis@ausil.us> - 4.1.3-2
- add patch to print more info for missing rpms
- add patch to clone repo with extra lorax templates for ostree_installer
* Fri Apr 08 2016 Dennis Gilmore <dennis@ausil.us> - 4.1.3-1 * Fri Apr 08 2016 Dennis Gilmore <dennis@ausil.us> - 4.1.3-1
- enable the compose test (dennis) - enable the compose test (dennis)
- [ostree-installer] Copy all lorax outputs (lsedlar) - [ostree-installer] Copy all lorax outputs (lsedlar)