almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

Merge #257 [ostree] Enable marking ostree phase as failable (ausil)

Browse Source 
- [ostree] Enable marking ostree phase as failable (lsedlar)
- [koji-wrapper] Initialize wrappers sequentially (lsedlar)
- [createiso] Simplify code, test phase (lsedlar)
- [createiso] Move runroot work to separate script (lsedlar)
- [ostree] Use explicit work directory (lsedlar)
- [ostree] Rename atomic to ostree (lsedlar)
- [ostree] Move cloning config repo to chroot (lsedlar)
- [ostree] Fix call to kobo.shortcuts.run (lsedlar)
- [atomic] Stop creating the os directory (lsedlar)
- [checksum] Add arch to file name (lsedlar)

Signed-off-by: Dennis Gilmore <dennis@ra.ausil.us>
This commit is contained in:
Dennis Gilmore 2016-04-06 18:09:58 -05:00
parent 32cfa08727
commit bcae962ab3
9 changed files with 16 additions and 2327 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

221
0001-checksum-Add-arch-to-file-name.patch
View File

@ -1,221 +0,0 @@
From e34bd2763c0d0776693f8842639b35b55dcb511b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
Date: Mon, 4 Apr 2016 09:38:48 +0200
Subject: [PATCH 1/6] [checksum] Add arch to file name
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
This allows the checksum file to include the architecture.
Fixes: #243
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
---
doc/configuration.rst | 16 +++++++++++++---
pungi/compose.py | 32 +++++++-------------------------
pungi/phases/image_checksum.py | 21 +++++++--------------
pungi/util.py | 35 ++++++++++++++++++++++++++++-------
tests/helpers.py | 2 ++
5 files changed, 57 insertions(+), 49 deletions(-)
diff --git a/doc/configuration.rst b/doc/configuration.rst
index 23bcbcf..7d4b91e 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -1051,9 +1051,19 @@ Media Checksums Settings
prefix to that name
It is possible to use format strings that will be replace by actual values.
- The allowed keys are ``%(release_showrt)s``, ``%(release_short)s``,
- ``%(release_id)s``, ``%(variant)s``, ``%(version)s``, ``%(date)s``,
- ``%(type_suffix)s``, ``%(label)s`` and ``%(respin)s``
+ The allowed keys are:
+
+ * ``arch``
+ * ``compose_id``
+ * ``date``
+ * ``label``
+ * ``label_major_version``
+ * ``release_short``
+ * ``respin``
+ * ``type``
+ * ``type_suffix``
+ * ``version``
+ * ``version``
For example, for Fedora the prefix should be
``%(release_short)s-%(variant)s-%(version)s-%(date)s%(type_suffix)s.%(respin)s``.
diff --git a/pungi/compose.py b/pungi/compose.py
index 8935a0f..f764129 100644
--- a/pungi/compose.py
+++ b/pungi/compose.py
@@ -33,7 +33,7 @@ from productmd.images import Images
from pungi.wrappers.variants import VariantsXmlParser
from pungi.paths import Paths
from pungi.wrappers.scm import get_file_from_scm
-from pungi.util import makedirs, get_arch_variant_data
+from pungi.util import makedirs, get_arch_variant_data, get_format_substs
from pungi.metadata import compose_to_composeinfo
@@ -270,25 +270,6 @@ class Compose(kobo.log.LoggingBase):
return
return open(self.status_file, "r").read().strip()
- def get_format_substs(self, **kwargs):
- """Return a dict of basic format substitutions.
-
- Any kwargs will be added as well.
- """
- substs = {
- 'compose_id': self.compose_id,
- 'release_short': self.ci_base.release.short,
- 'version': self.ci_base.release.version,
- 'date': self.compose_date,
- 'respin': self.compose_respin,
- 'type': self.compose_type,
- 'type_suffix': self.compose_type_suffix,
- 'label': self.compose_label,
- 'label_major_version': self.compose_label_major_version,
- }
- substs.update(kwargs)
- return substs
-
def get_image_name(self, arch, variant, disc_type='dvd',
disc_num=1, suffix='.iso', format=None):
"""Create a filename for image with given parameters.
@@ -310,11 +291,12 @@ class Compose(kobo.log.LoggingBase):
variant_uid = variant.parent.uid
else:
variant_uid = variant.uid
- args = self.get_format_substs(variant=variant_uid,
- arch=arch,
- disc_type=disc_type,
- disc_num=disc_num,
- suffix=suffix)
+ args = get_format_substs(self,
+ variant=variant_uid,
+ arch=arch,
+ disc_type=disc_type,
+ disc_num=disc_num,
+ suffix=suffix)
try:
return format % args
except KeyError as err:
diff --git a/pungi/phases/image_checksum.py b/pungi/phases/image_checksum.py
index c9ef321..aa74c4f 100644
--- a/pungi/phases/image_checksum.py
+++ b/pungi/phases/image_checksum.py
@@ -4,6 +4,7 @@ import os
from kobo import shortcuts
from .base import PhaseBase
+from ..util import get_format_substs
MULTIPLE_CHECKSUMS_ERROR = (
@@ -66,29 +67,21 @@ class ImageChecksumPhase(PhaseBase):
for arch in self.compose.im.images[variant]:
for image in self.compose.im.images[variant][arch]:
path = os.path.dirname(os.path.join(top_dir, image.path))
- images.setdefault((variant, path), set()).add(image)
+ images.setdefault((variant, arch, path), set()).add(image)
return images
- def _get_base_filename(self, variant):
+ def _get_base_filename(self, variant, arch):
base_checksum_name = self.compose.conf.get('media_checksum_base_filename', '')
if base_checksum_name:
- base_checksum_name = base_checksum_name % {
- 'release_short': self.compose.ci_base.release.short,
- 'release_id': self.compose.ci_base.release_id,
- 'variant': variant,
- 'version': self.compose.ci_base.release.version,
- 'date': self.compose.compose_date,
- 'type_suffix': self.compose.compose_type_suffix,
- 'respin': self.compose.compose_respin,
- 'label': self.compose.compose_label,
- }
+ substs = get_format_substs(self.compose, variant=variant, arch=arch)
+ base_checksum_name = base_checksum_name % substs
base_checksum_name += '-'
return base_checksum_name
def run(self):
- for (variant, path), images in self._get_images().iteritems():
+ for (variant, arch, path), images in self._get_images().iteritems():
checksums = {}
- base_checksum_name = self._get_base_filename(variant)
+ base_checksum_name = self._get_base_filename(variant, arch)
for image in images:
filename = os.path.basename(image.path)
full_path = os.path.join(path, filename)
diff --git a/pungi/util.py b/pungi/util.py
index f4f5e33..6a6ccf3 100644
--- a/pungi/util.py
+++ b/pungi/util.py
@@ -370,13 +370,14 @@ def get_volid(compose, arch, variant=None, escape_spaces=False, disc_type=False)
if not variant_uid and "%(variant)s" in i:
continue
try:
- volid = i % compose.get_format_substs(variant=variant_uid,
- release_short=release_short,
- version=release_version,
- arch=arch,
- disc_type=disc_type or '',
- base_product_short=base_product_short,
- base_product_version=base_product_version)
+ volid = i % get_format_substs(compose,
+ variant=variant_uid,
+ release_short=release_short,
+ version=release_version,
+ arch=arch,
+ disc_type=disc_type or '',
+ base_product_short=base_product_short,
+ base_product_version=base_product_version)
except KeyError as err:
raise RuntimeError('Failed to create volume id: unknown format element: %s' % err.message)
volid = _apply_substitutions(compose, volid)
@@ -472,3 +473,23 @@ def failable(compose, variant, arch, deliverable, msg=None):
compose.log_info(str(exc))
tb = traceback.format_exc()
compose.log_debug(tb)
+
+
+def get_format_substs(compose, **kwargs):
+ """Return a dict of basic format substitutions.
+
+ Any kwargs will be added as well.
+ """
+ substs = {
+ 'compose_id': compose.compose_id,
+ 'release_short': compose.ci_base.release.short,
+ 'version': compose.ci_base.release.version,
+ 'date': compose.compose_date,
+ 'respin': compose.compose_respin,
+ 'type': compose.compose_type,
+ 'type_suffix': compose.compose_type_suffix,
+ 'label': compose.compose_label,
+ 'label_major_version': compose.compose_label_major_version,
+ }
+ substs.update(kwargs)
+ return substs
diff --git a/tests/helpers.py b/tests/helpers.py
index 9a0983c..11c07a4 100644
--- a/tests/helpers.py
+++ b/tests/helpers.py
@@ -23,9 +23,11 @@ class DummyCompose(object):
self.supported = True
self.compose_date = '20151203'
self.compose_type_suffix = '.t'
+ self.compose_type = 'test'
self.compose_respin = 0
self.compose_id = 'Test-20151203.0.t'
self.compose_label = None
+ self.compose_label_major_version = None
self.image_release = '20151203.t.0'
self.ci_base = mock.Mock(
release_id='Test-1.0',
--
2.7.3

42
0002-atomic-Stop-creating-the-os-directory.patch
View File

@ -1,42 +0,0 @@
From f18e32c5affad9bf376103a536c7ded19366d92f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
Date: Mon, 4 Apr 2016 14:24:18 +0200
Subject: [PATCH 2/6] [atomic] Stop creating the os directory
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The output directory for Lorax can not exist beforehand, or the process
will fail.
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
---
pungi/phases/atomic_installer.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/pungi/phases/atomic_installer.py b/pungi/phases/atomic_installer.py
index 476ad1c..3390a74 100644
--- a/pungi/phases/atomic_installer.py
+++ b/pungi/phases/atomic_installer.py
@@ -49,7 +49,8 @@ class AtomicInstallerThread(WorkerThread):
self.logdir = compose.paths.log.topdir('{}/atomic'.format(arch))
source_variant = compose.variants[config['source_repo_from']]
- source_repo = translate_path(compose, compose.paths.compose.repository(arch, source_variant))
+ source_repo = translate_path(
+ compose, compose.paths.compose.repository(arch, source_variant, create_dir=False))
self._run_atomic_cmd(compose, variant, arch, config, source_repo)
@@ -100,7 +101,7 @@ class AtomicInstallerThread(WorkerThread):
compose.im.add(variant.uid, arch, img)
def _run_atomic_cmd(self, compose, variant, arch, config, source_repo):
- image_dir = compose.paths.compose.os_tree(arch, variant)
+ image_dir = compose.paths.compose.os_tree(arch, variant, create_dir=False)
lorax_wrapper = lorax.LoraxWrapper()
cmd = lorax_wrapper.get_lorax_cmd(
compose.conf['release_name'],
--
2.7.3

54
0003-ostree-Fix-call-to-kobo.shortcuts.run.patch
View File

@ -1,54 +0,0 @@
From 05384eae584476bfca3bd4bf31958d1e5dbb20a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
Date: Mon, 4 Apr 2016 15:13:35 +0200
Subject: [PATCH 3/6] [ostree] Fix call to kobo.shortcuts.run
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
---
pungi/atomic.py | 4 ++--
tests/test_atomic_script.py | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/pungi/atomic.py b/pungi/atomic.py
index 95ccfe0..7c0075b 100644
--- a/pungi/atomic.py
+++ b/pungi/atomic.py
@@ -22,13 +22,13 @@ def init_atomic_repo(repo, log_dir=None):
log_file = make_log_file(log_dir, 'init-atomic-repo')
if not os.path.isdir(repo):
shortcuts.run(['ostree', 'init', '--repo={}'.format(repo), '--mode=archive-z2'],
- log_file=log_file)
+ logfile=log_file)
def make_ostree_repo(repo, config, log_dir=None):
log_file = make_log_file(log_dir, 'create-atomic-repo')
shortcuts.run(['rpm-ostree', 'compose', 'tree', '--repo={}'.format(repo), config],
- log_file=log_file)
+ logfile=log_file)
def run(opts):
diff --git a/tests/test_atomic_script.py b/tests/test_atomic_script.py
index 13aa825..a871659 100755
--- a/tests/test_atomic_script.py
+++ b/tests/test_atomic_script.py
@@ -28,10 +28,10 @@ class OstreeScriptTest(helpers.PungiTestCase):
self.assertItemsEqual(
run.call_args_list,
[mock.call(['ostree', 'init', '--repo={}/atomic'.format(self.topdir), '--mode=archive-z2'],
- log_file=self.topdir + '/logs/Atomic/init-atomic-repo.log'),
+ logfile=self.topdir + '/logs/Atomic/init-atomic-repo.log'),
mock.call(['rpm-ostree', 'compose', 'tree', '--repo={}/atomic'.format(self.topdir),
self.topdir + '/work/fedora-atomic-docker-host.json'],
- log_file=self.topdir + '/logs/Atomic/create-atomic-repo.log')])
+ logfile=self.topdir + '/logs/Atomic/create-atomic-repo.log')])
if __name__ == '__main__':
--
2.7.3

276
0004-ostree-Move-cloning-config-repo-to-chroot.patch
View File

@ -1,276 +0,0 @@
From 446d4b20cc508ae466a1c5bf9cc76f8b0fc12494 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
Date: Tue, 5 Apr 2016 08:40:54 +0200
Subject: [PATCH 4/6] [ostree] Move cloning config repo to chroot
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
---
pungi/atomic.py | 46 ++++++++++++++++++++++++++++++++++++++++++++-
pungi/phases/ostree.py | 29 ++++------------------------
tests/helpers.py | 7 ++++++-
tests/test_atomic_script.py | 23 ++++++++++++++++++++---
tests/test_ostree_phase.py | 15 ++++-----------
5 files changed, 79 insertions(+), 41 deletions(-)
diff --git a/pungi/atomic.py b/pungi/atomic.py
index 7c0075b..627ce34 100644
--- a/pungi/atomic.py
+++ b/pungi/atomic.py
@@ -8,6 +8,11 @@ It is expected to be runnable in Koji runroot.
import argparse
import os
from kobo import shortcuts
+import tempfile
+import shutil
+import re
+
+from .wrappers import scm
def make_log_file(log_dir, filename):
@@ -31,9 +36,42 @@ def make_ostree_repo(repo, config, log_dir=None):
logfile=log_file)
+def clone_repo(repodir, url, branch):
+ scm.get_dir_from_scm(
+ {'scm': 'git', 'repo': url, 'branch': branch, 'dir': '.'}, repodir)
+
+
+def tweak_mirrorlist(repodir, source_repo):
+ for file in os.listdir(repodir):
+ if file.endswith('.repo'):
+ tweak_file(os.path.join(repodir, file), source_repo)
+
+
+def tweak_file(path, source_repo):
+ """Replace mirrorlist line in repo file with baseurl pointing to source_repo."""
+ with open(path, 'r') as f:
+ contents = f.read()
+ replacement = 'baseurl={}'.format(source_repo)
+ contents = re.sub(r'^mirrorlist=.*$', replacement, contents)
+ with open(path, 'w') as f:
+ f.write(contents)
+
+
+def prepare_config(workdir, config_url, config_branch, source_repo):
+ repodir = os.path.join(workdir, 'config_repo')
+ clone_repo(repodir, config_url, config_branch)
+ tweak_mirrorlist(repodir, source_repo)
+ return repodir
+
+
def run(opts):
+ workdir = tempfile.mkdtemp()
+ repodir = prepare_config(workdir, opts.config_url, opts.config_branch,
+ opts.source_repo)
init_atomic_repo(opts.atomic_repo, log_dir=opts.log_dir)
- make_ostree_repo(opts.atomic_repo, opts.treefile, log_dir=opts.log_dir)
+ treefile = os.path.join(repodir, opts.treefile)
+ make_ostree_repo(opts.atomic_repo, treefile, log_dir=opts.log_dir)
+ shutil.rmtree(workdir)
def main(args=None):
@@ -45,6 +83,12 @@ def main(args=None):
help='where to put the atomic repo')
parser.add_argument('--treefile', required=True,
help='treefile for rpm-ostree')
+ parser.add_argument('--config-url', required=True,
+ help='git repository with the treefile')
+ parser.add_argument('--config-branch', default='master',
+ help='git branch to be used')
+ parser.add_argument('--source-repo', required=True,
+ help='yum repo used as source for')
opts = parser.parse_args(args)
diff --git a/pungi/phases/ostree.py b/pungi/phases/ostree.py
index 36cbc97..79e58d4 100644
--- a/pungi/phases/ostree.py
+++ b/pungi/phases/ostree.py
@@ -2,12 +2,11 @@
import os
from kobo.threads import ThreadPool, WorkerThread
-import re
from .base import ConfigGuardedPhase
from .. import util
from ..paths import translate_path
-from ..wrappers import scm, kojiwrapper
+from ..wrappers import kojiwrapper
class OSTreePhase(ConfigGuardedPhase):
@@ -42,15 +41,11 @@ class OSTreeThread(WorkerThread):
msg = 'OSTree phase for variant %s, arch %s' % (variant.uid, arch)
self.pool.log_info('[BEGIN] %s' % msg)
- workdir = compose.paths.work.topdir('atomic')
self.logdir = compose.paths.log.topdir('{}/atomic'.format(arch))
- repodir = os.path.join(workdir, 'config_repo')
source_variant = compose.variants[config['source_repo_from']]
source_repo = translate_path(compose, compose.paths.compose.repository(arch, source_variant))
- self._clone_repo(repodir, config['config_url'], config.get('config_branch', 'master'))
- self._tweak_mirrorlist(repodir, source_repo)
self._run_atomic_cmd(compose, variant, arch, config, source_repo)
self.pool.log_info('[DONE ] %s' % msg)
@@ -60,6 +55,9 @@ class OSTreeThread(WorkerThread):
'pungi-make-ostree',
'--log-dir={}'.format(self.logdir),
'--treefile={}'.format(config['treefile']),
+ '--config-url={}'.format(config['config_url']),
+ '--config-branch={}'.format(config.get('config_branch', 'master')),
+ '--source-repo={}'.format(source_repo),
config['atomic_repo']
]
@@ -77,22 +75,3 @@ class OSTreeThread(WorkerThread):
if output["retcode"] != 0:
raise RuntimeError("Runroot task failed: %s. See %s for more details."
% (output["task_id"], log_file))
-
- def _clone_repo(self, repodir, url, branch):
- scm.get_dir_from_scm({'scm': 'git', 'repo': url, 'branch': branch, 'dir': '.'},
- repodir, logger=self.pool._logger)
-
- def _tweak_mirrorlist(self, repodir, source_repo):
- for file in os.listdir(repodir):
- if file.endswith('.repo'):
- tweak_file(os.path.join(repodir, file), source_repo)
-
-
-def tweak_file(path, source_repo):
- """Replace mirrorlist line in repo file with baseurl pointing to source_repo."""
- with open(path, 'r') as f:
- contents = f.read()
- replacement = 'baseurl={}'.format(source_repo)
- contents = re.sub(r'^mirrorlist=.*$', replacement, contents)
- with open(path, 'w') as f:
- f.write(contents)
diff --git a/tests/helpers.py b/tests/helpers.py
index 11c07a4..d93e095 100644
--- a/tests/helpers.py
+++ b/tests/helpers.py
@@ -5,6 +5,7 @@ import os
import unittest
import tempfile
import shutil
+import errno
from pungi.util import get_arch_variant_data
from pungi import paths
@@ -15,7 +16,11 @@ class PungiTestCase(unittest.TestCase):
self.topdir = tempfile.mkdtemp()
def tearDown(self):
- shutil.rmtree(self.topdir)
+ try:
+ shutil.rmtree(self.topdir)
+ except OSError as err:
+ if err.errno != errno.ENOENT:
+ raise
class DummyCompose(object):
diff --git a/tests/test_atomic_script.py b/tests/test_atomic_script.py
index a871659..4b0b396 100755
--- a/tests/test_atomic_script.py
+++ b/tests/test_atomic_script.py
@@ -17,20 +17,37 @@ from pungi import atomic
class OstreeScriptTest(helpers.PungiTestCase):
+ def _dummy_config_repo(self, scm_dict, target, logger=None):
+ helpers.touch(os.path.join(target, 'fedora-atomic-docker-host.json'))
+ helpers.touch(os.path.join(target, 'fedora-rawhide.repo'))
+
+ @mock.patch('tempfile.mkdtemp')
@mock.patch('kobo.shortcuts.run')
- def test_full_run(self, run):
+ @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
+ def test_full_run(self, get_dir_from_scm, run, tempfile):
+ tempfile.return_value = self.topdir
+ get_dir_from_scm.side_effect = self._dummy_config_repo
+
atomic.main([
'--log-dir={}'.format(os.path.join(self.topdir, 'logs', 'Atomic')),
- '--treefile={}'.format(os.path.join(self.topdir, 'work', 'fedora-atomic-docker-host.json')),
+ '--treefile={}'.format('fedora-atomic-docker-host.json'),
+ '--config-url=https://git.fedorahosted.org/git/fedora-atomic.git',
+ '--config-branch=f24',
+ '--source-repo=https://kojipkgs.fedoraproject.org/repo',
os.path.join(self.topdir, 'atomic'),
])
+
self.maxDiff = None
+ self.assertEqual(get_dir_from_scm.call_args_list,
+ [mock.call({'scm': 'git', 'repo': 'https://git.fedorahosted.org/git/fedora-atomic.git',
+ 'branch': 'f24', 'dir': '.'},
+ self.topdir + '/config_repo')])
self.assertItemsEqual(
run.call_args_list,
[mock.call(['ostree', 'init', '--repo={}/atomic'.format(self.topdir), '--mode=archive-z2'],
logfile=self.topdir + '/logs/Atomic/init-atomic-repo.log'),
mock.call(['rpm-ostree', 'compose', 'tree', '--repo={}/atomic'.format(self.topdir),
- self.topdir + '/work/fedora-atomic-docker-host.json'],
+ self.topdir + '/config_repo/fedora-atomic-docker-host.json'],
logfile=self.topdir + '/logs/Atomic/create-atomic-repo.log')])
diff --git a/tests/test_ostree_phase.py b/tests/test_ostree_phase.py
index e4664e1..fe0036c 100755
--- a/tests/test_ostree_phase.py
+++ b/tests/test_ostree_phase.py
@@ -74,13 +74,8 @@ class OSTreePhaseTest(helpers.PungiTestCase):
class OSTreeThreadTest(helpers.PungiTestCase):
- def _dummy_config_repo(self, scm_dict, target, logger=None):
- helpers.touch(os.path.join(target, 'fedora-atomic-docker-host.json'))
- helpers.touch(os.path.join(target, 'fedora-rawhide.repo'))
-
- @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
@mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
- def test_run(self, KojiWrapper, get_dir_from_scm):
+ def test_run(self, KojiWrapper):
compose = helpers.DummyCompose(self.topdir, {
'koji_profile': 'koji',
'runroot_tag': 'rrt',
@@ -93,7 +88,6 @@ class OSTreeThreadTest(helpers.PungiTestCase):
'treefile': 'fedora-atomic-docker-host.json',
'atomic_repo': '/other/place/for/atomic'
}
- get_dir_from_scm.side_effect = self._dummy_config_repo
koji = KojiWrapper.return_value
koji.run_runroot_cmd.return_value = {
'task_id': 1234,
@@ -105,15 +99,14 @@ class OSTreeThreadTest(helpers.PungiTestCase):
t.process((compose, compose.variants['Everything'], 'x86_64', cfg), 1)
- self.assertEqual(get_dir_from_scm.call_args_list,
- [mock.call({'scm': 'git', 'repo': 'https://git.fedorahosted.org/git/fedora-atomic.git',
- 'branch': 'f24', 'dir': '.'},
- self.topdir + '/work/atomic/config_repo', logger=pool._logger)])
self.assertEqual(koji.get_runroot_cmd.call_args_list,
[mock.call('rrt', 'x86_64',
['pungi-make-ostree',
'--log-dir={}/logs/x86_64/atomic'.format(self.topdir),
'--treefile=fedora-atomic-docker-host.json',
+ '--config-url=https://git.fedorahosted.org/git/fedora-atomic.git',
+ '--config-branch=f24',
+ '--source-repo={}/compose/Everything/x86_64/os'.format(self.topdir),
'/other/place/for/atomic'],
channel=None, mounts=[self.topdir],
packages=['pungi', 'ostree', 'rpm-ostree'],
--
2.7.3

1494
0005-ostree-Rename-atomic-to-ostree.patch
View File

File diff suppressed because it is too large Load Diff

168
0006-ostree-Use-explicit-work-directory.patch
View File

@ -1,168 +0,0 @@
From 79ee9a4ffaf4627eade703c9bd819c870258745f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
Date: Tue, 5 Apr 2016 12:46:59 +0200
Subject: [PATCH 6/6] [ostree] Use explicit work directory
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
---
pungi/ostree.py | 26 +++++++++++++++++++-------
pungi/phases/ostree.py | 2 ++
tests/test_ostree_phase.py | 1 +
tests/test_ostree_script.py | 9 ++++-----
4 files changed, 26 insertions(+), 12 deletions(-)
diff --git a/pungi/ostree.py b/pungi/ostree.py
index de2cfcd..c0c8a04 100644
--- a/pungi/ostree.py
+++ b/pungi/ostree.py
@@ -8,17 +8,26 @@ It is expected to be runnable in Koji runroot.
import argparse
import os
from kobo import shortcuts
-import tempfile
-import shutil
import re
+import errno
from .wrappers import scm
+def ensure_dir(path):
+ try:
+ os.makedirs(path)
+ except OSError as err:
+ if err.errno != errno.EEXIST:
+ raise
+ return path
+
+
def make_log_file(log_dir, filename):
"""Return path to log file with given name, if log_dir is set."""
if not log_dir:
return None
+ ensure_dir(log_dir)
return os.path.join(log_dir, '{}.log'.format(filename))
@@ -26,14 +35,15 @@ def init_ostree_repo(repo, log_dir=None):
"""If the ostree repo does not exist, initialize it."""
log_file = make_log_file(log_dir, 'init-ostree-repo')
if not os.path.isdir(repo):
+ ensure_dir(repo)
shortcuts.run(['ostree', 'init', '--repo={}'.format(repo), '--mode=archive-z2'],
- logfile=log_file)
+ show_cmd=True, logfile=log_file)
def make_ostree_repo(repo, config, log_dir=None):
log_file = make_log_file(log_dir, 'create-ostree-repo')
shortcuts.run(['rpm-ostree', 'compose', 'tree', '--repo={}'.format(repo), config],
- logfile=log_file)
+ show_cmd=True, logfile=log_file)
def clone_repo(repodir, url, branch):
@@ -52,7 +62,7 @@ def tweak_file(path, source_repo):
with open(path, 'r') as f:
contents = f.read()
replacement = 'baseurl={}'.format(source_repo)
- contents = re.sub(r'^mirrorlist=.*$', replacement, contents)
+ contents = re.sub(r'^mirrorlist=.*$', replacement, contents, flags=re.MULTILINE)
with open(path, 'w') as f:
f.write(contents)
@@ -65,19 +75,21 @@ def prepare_config(workdir, config_url, config_branch, source_repo):
def run(opts):
- workdir = tempfile.mkdtemp()
+ workdir = ensure_dir(opts.work_dir)
repodir = prepare_config(workdir, opts.config_url, opts.config_branch,
opts.source_repo)
+ ensure_dir(repodir)
init_ostree_repo(opts.ostree_repo, log_dir=opts.log_dir)
treefile = os.path.join(repodir, opts.treefile)
make_ostree_repo(opts.ostree_repo, treefile, log_dir=opts.log_dir)
- shutil.rmtree(workdir)
def main(args=None):
parser = argparse.ArgumentParser()
parser.add_argument('--log-dir',
help='where to log output')
+ parser.add_argument('--work-dir', required=True,
+ help='where to put temporary files')
parser.add_argument('ostree_repo', metavar='OSTREE_REPO',
help='where to put the ostree repo')
diff --git a/pungi/phases/ostree.py b/pungi/phases/ostree.py
index 810716e..77e4838 100644
--- a/pungi/phases/ostree.py
+++ b/pungi/phases/ostree.py
@@ -51,9 +51,11 @@ class OSTreeThread(WorkerThread):
self.pool.log_info('[DONE ] %s' % msg)
def _run_ostree_cmd(self, compose, variant, arch, config, source_repo):
+ workdir = os.path.join(compose.paths.work.topdir(arch), 'ostree')
cmd = [
'pungi-make-ostree',
'--log-dir={}'.format(self.logdir),
+ '--work-dir={}'.format(workdir),
'--treefile={}'.format(config['treefile']),
'--config-url={}'.format(config['config_url']),
'--config-branch={}'.format(config.get('config_branch', 'master')),
diff --git a/tests/test_ostree_phase.py b/tests/test_ostree_phase.py
index 9f8fdf1..7617dde 100755
--- a/tests/test_ostree_phase.py
+++ b/tests/test_ostree_phase.py
@@ -103,6 +103,7 @@ class OSTreeThreadTest(helpers.PungiTestCase):
[mock.call('rrt', 'x86_64',
['pungi-make-ostree',
'--log-dir={}/logs/x86_64/ostree'.format(self.topdir),
+ '--work-dir={}/work/x86_64/ostree'.format(self.topdir),
'--treefile=fedora-atomic-docker-host.json',
'--config-url=https://git.fedorahosted.org/git/fedora-atomic.git',
'--config-branch=f24',
diff --git a/tests/test_ostree_script.py b/tests/test_ostree_script.py
index 744af4d..7b59680 100755
--- a/tests/test_ostree_script.py
+++ b/tests/test_ostree_script.py
@@ -21,17 +21,16 @@ class OstreeScriptTest(helpers.PungiTestCase):
helpers.touch(os.path.join(target, 'fedora-atomic-docker-host.json'))
helpers.touch(os.path.join(target, 'fedora-rawhide.repo'))
- @mock.patch('tempfile.mkdtemp')
@mock.patch('kobo.shortcuts.run')
@mock.patch('pungi.wrappers.scm.get_dir_from_scm')
- def test_full_run(self, get_dir_from_scm, run, tempfile):
- tempfile.return_value = self.topdir
+ def test_full_run(self, get_dir_from_scm, run):
get_dir_from_scm.side_effect = self._dummy_config_repo
repo = os.path.join(self.topdir, 'atomic')
ostree.main([
'--log-dir={}'.format(os.path.join(self.topdir, 'logs', 'Atomic')),
+ '--work-dir={}'.format(self.topdir),
'--treefile={}'.format('fedora-atomic-docker-host.json'),
'--config-url=https://git.fedorahosted.org/git/fedora-atomic.git',
'--config-branch=f24',
@@ -47,10 +46,10 @@ class OstreeScriptTest(helpers.PungiTestCase):
self.assertItemsEqual(
run.call_args_list,
[mock.call(['ostree', 'init', '--repo={}'.format(repo), '--mode=archive-z2'],
- logfile=self.topdir + '/logs/Atomic/init-ostree-repo.log'),
+ logfile=self.topdir + '/logs/Atomic/init-ostree-repo.log', show_cmd=True),
mock.call(['rpm-ostree', 'compose', 'tree', '--repo={}'.format(repo),
self.topdir + '/config_repo/fedora-atomic-docker-host.json'],
- logfile=self.topdir + '/logs/Atomic/create-ostree-repo.log')])
+ logfile=self.topdir + '/logs/Atomic/create-ostree-repo.log', show_cmd=True)])
if __name__ == '__main__':
--
2.7.3

54
c52cb198c5ffc0593720c927f469793bedb3d4a4.patch
View File

@ -1,54 +0,0 @@
From c52cb198c5ffc0593720c927f469793bedb3d4a4 Mon Sep 17 00:00:00 2001
From: Peter Robinson <pbrobinson@gmail.com>
Date: Mar 16 2016 22:03:33 +0000
Subject: Fix locations of ppc lorax templates for PowerPC iso creation
PowerPC needs a mapping file for iso creation, with lorax moving the
templates locations in 24+ we need to update this so runroot iso
creation doesn't fail.
Signed-off-by: Peter Robinson <pbrobinson@fedoraproject.org>
---
diff --git a/pungi/gather.py b/pungi/gather.py
index 2e5048c..c27070f 100644
--- a/pungi/gather.py
+++ b/pungi/gather.py
@@ -1590,7 +1590,7 @@ class Pungi(PungiBase):
return
isolist = []
- ppcbootinfo = '/usr/share/lorax/config_files/ppc'
+ ppcbootinfo = '/usr/share/lorax/templates.d/99-generic/config_files/ppc'
pungi.util._ensuredir(self.isodir, self.logger,
force=self.config.getboolean('pungi', 'force'),
diff --git a/pungi/phases/createiso.py b/pungi/phases/createiso.py
index f88b20d..eab3133 100644
--- a/pungi/phases/createiso.py
+++ b/pungi/phases/createiso.py
@@ -142,7 +142,7 @@ class CreateisoPhase(PhaseBase):
buildinstall_method = self.compose.conf["buildinstall_method"]
if buildinstall_method == "lorax":
# TODO: $arch instead of ppc
- mkisofs_kwargs["boot_args"] = iso.get_boot_options(arch, "/usr/share/lorax/config_files/ppc")
+ mkisofs_kwargs["boot_args"] = iso.get_boot_options(arch, "/usr/share/lorax/templates.d/99-generic/config_files/ppc")
elif buildinstall_method == "buildinstall":
mkisofs_kwargs["boot_args"] = iso.get_boot_options(arch, "/usr/lib/anaconda-runtime/boot")
diff --git a/pungi/phases/product_img.py b/pungi/phases/product_img.py
index 687d261..8da4e23 100644
--- a/pungi/phases/product_img.py
+++ b/pungi/phases/product_img.py
@@ -230,7 +230,7 @@ def rebuild_boot_iso(compose, arch, variant, package_sets):
boot_files = None
if buildinstall_method == "lorax":
# TODO: $arch instead of ppc
- mkisofs_kwargs["boot_args"] = iso.get_boot_options(arch, "/usr/share/lorax/config_files/ppc")
+ mkisofs_kwargs["boot_args"] = iso.get_boot_options(arch, "/usr/share/lorax/templates.d/99-generic/config_files/ppc")
elif buildinstall_method == "buildinstall":
boot_files = explode_anaconda(compose, arch, variant, package_sets)
mkisofs_kwargs["boot_args"] = iso.get_boot_options(arch, boot_files)

32
pungi.spec
View File

@ -1,20 +1,12 @@
Name: pungi Name: pungi
Version: 4.1.1 Version: 4.1.2
Release: 3%{?dist} Release: 1%{?dist}
Summary: Distribution compose tool Summary: Distribution compose tool
Group: Development/Tools Group: Development/Tools
License: GPLv2 License: GPLv2
URL: https://pagure.io/pungi URL: https://pagure.io/pungi
Source0: https://fedorahosted.org/pungi/attachment/wiki/%{version}/%{name}-%{version}.tar.bz2 Source0: https://fedorahosted.org/pungi/attachment/wiki/%{version}/%{name}-%{version}.tar.bz2
Patch0: 0001-checksum-Add-arch-to-file-name.patch
Patch1: 0002-atomic-Stop-creating-the-os-directory.patch
Patch2: 0003-ostree-Fix-call-to-kobo.shortcuts.run.patch
Patch3: 0004-ostree-Move-cloning-config-repo-to-chroot.patch
Patch4: 0005-ostree-Rename-atomic-to-ostree.patch
Patch5: 0006-ostree-Use-explicit-work-directory.patch
# bandaid for https://pagure.io/pungi/issue/231 until we get things fixed properly upstream
Patch6: c52cb198c5ffc0593720c927f469793bedb3d4a4.patch
BuildRequires: python-nose, python-nose-cov, python-mock BuildRequires: python-nose, python-nose-cov, python-mock
BuildRequires: python-devel, python-setuptools, python2-productmd BuildRequires: python-devel, python-setuptools, python2-productmd
@ -56,13 +48,6 @@ A tool to create anaconda based installation trees/isos of a set of rpms.
%prep %prep
%setup -q %setup -q
%patch0 -p1
%patch1 -p1
%patch2 -p1
%patch3 -p1
%patch4 -p1
%patch5 -p1
%patch6 -p1
%build %build
%{__python} setup.py build %{__python} setup.py build
@ -96,6 +81,19 @@ cd tests && ./test_compose.sh
/var/cache/pungi /var/cache/pungi
%changelog %changelog
* Wed Apr 05 2016 Dennis Gilmore <dennis@ausil.us> - 4.1.2-1
- Merge #257 `[ostree] Enable marking ostree phase as failable` (ausil)
- [ostree] Enable marking ostree phase as failable (lsedlar)
- [koji-wrapper] Initialize wrappers sequentially (lsedlar)
- [createiso] Simplify code, test phase (lsedlar)
- [createiso] Move runroot work to separate script (lsedlar)
- [ostree] Use explicit work directory (lsedlar)
- [ostree] Rename atomic to ostree (lsedlar)
- [ostree] Move cloning config repo to chroot (lsedlar)
- [ostree] Fix call to kobo.shortcuts.run (lsedlar)
- [atomic] Stop creating the os directory (lsedlar)
- [checksum] Add arch to file name (lsedlar)
* Tue Apr 05 2016 Dennis Gilmore <dennis@ausil.us> - 4.1.1-3 * Tue Apr 05 2016 Dennis Gilmore <dennis@ausil.us> - 4.1.1-3
- add some more ostree fixes - add some more ostree fixes
- add a bandaid for ppc until we get a proper fix - add a bandaid for ppc until we get a proper fix

2
sources
View File

@ -1 +1 @@
b53d104d3e4455cb1cf98c790e8d0ca1 pungi-4.1.1.tar.bz2 4b63f8b87a43da6f3946f537c7b2f83c pungi-4.1.2.tar.bz2