Merge #257 [ostree] Enable marking ostree phase as failable (ausil)
- [ostree] Enable marking ostree phase as failable (lsedlar) - [koji-wrapper] Initialize wrappers sequentially (lsedlar) - [createiso] Simplify code, test phase (lsedlar) - [createiso] Move runroot work to separate script (lsedlar) - [ostree] Use explicit work directory (lsedlar) - [ostree] Rename atomic to ostree (lsedlar) - [ostree] Move cloning config repo to chroot (lsedlar) - [ostree] Fix call to kobo.shortcuts.run (lsedlar) - [atomic] Stop creating the os directory (lsedlar) - [checksum] Add arch to file name (lsedlar) Signed-off-by: Dennis Gilmore <dennis@ra.ausil.us>
This commit is contained in:
Dennis Gilmore
parent
32cfa08727
commit
bcae962ab3
@ -1,221 +0,0 @@
|
||||
From e34bd2763c0d0776693f8842639b35b55dcb511b Mon Sep 17 00:00:00 2001
|
||||
From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
|
||||
Date: Mon, 4 Apr 2016 09:38:48 +0200
|
||||
Subject: [PATCH 1/6] [checksum] Add arch to file name
|
||||
MIME-Version: 1.0
|
||||
Content-Type: text/plain; charset=UTF-8
|
||||
Content-Transfer-Encoding: 8bit
|
||||
|
||||
This allows the checksum file to include the architecture.
|
||||
|
||||
Fixes: #243
|
||||
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
|
||||
---
|
||||
doc/configuration.rst | 16 +++++++++++++---
|
||||
pungi/compose.py | 32 +++++++-------------------------
|
||||
pungi/phases/image_checksum.py | 21 +++++++--------------
|
||||
pungi/util.py | 35 ++++++++++++++++++++++++++++-------
|
||||
tests/helpers.py | 2 ++
|
||||
5 files changed, 57 insertions(+), 49 deletions(-)
|
||||
|
||||
diff --git a/doc/configuration.rst b/doc/configuration.rst
|
||||
index 23bcbcf..7d4b91e 100644
|
||||
--- a/doc/configuration.rst
|
||||
+++ b/doc/configuration.rst
|
||||
@@ -1051,9 +1051,19 @@ Media Checksums Settings
|
||||
prefix to that name
|
||||
|
||||
It is possible to use format strings that will be replace by actual values.
|
||||
- The allowed keys are ``%(release_showrt)s``, ``%(release_short)s``,
|
||||
- ``%(release_id)s``, ``%(variant)s``, ``%(version)s``, ``%(date)s``,
|
||||
- ``%(type_suffix)s``, ``%(label)s`` and ``%(respin)s``
|
||||
+ The allowed keys are:
|
||||
+
|
||||
+ * ``arch``
|
||||
+ * ``compose_id``
|
||||
+ * ``date``
|
||||
+ * ``label``
|
||||
+ * ``label_major_version``
|
||||
+ * ``release_short``
|
||||
+ * ``respin``
|
||||
+ * ``type``
|
||||
+ * ``type_suffix``
|
||||
+ * ``version``
|
||||
+ * ``version``
|
||||
|
||||
For example, for Fedora the prefix should be
|
||||
``%(release_short)s-%(variant)s-%(version)s-%(date)s%(type_suffix)s.%(respin)s``.
|
||||
diff --git a/pungi/compose.py b/pungi/compose.py
|
||||
index 8935a0f..f764129 100644
|
||||
--- a/pungi/compose.py
|
||||
+++ b/pungi/compose.py
|
||||
@@ -33,7 +33,7 @@ from productmd.images import Images
|
||||
from pungi.wrappers.variants import VariantsXmlParser
|
||||
from pungi.paths import Paths
|
||||
from pungi.wrappers.scm import get_file_from_scm
|
||||
-from pungi.util import makedirs, get_arch_variant_data
|
||||
+from pungi.util import makedirs, get_arch_variant_data, get_format_substs
|
||||
from pungi.metadata import compose_to_composeinfo
|
||||
|
||||
|
||||
@@ -270,25 +270,6 @@ class Compose(kobo.log.LoggingBase):
|
||||
return
|
||||
return open(self.status_file, "r").read().strip()
|
||||
|
||||
- def get_format_substs(self, **kwargs):
|
||||
- """Return a dict of basic format substitutions.
|
||||
-
|
||||
- Any kwargs will be added as well.
|
||||
- """
|
||||
- substs = {
|
||||
- 'compose_id': self.compose_id,
|
||||
- 'release_short': self.ci_base.release.short,
|
||||
- 'version': self.ci_base.release.version,
|
||||
- 'date': self.compose_date,
|
||||
- 'respin': self.compose_respin,
|
||||
- 'type': self.compose_type,
|
||||
- 'type_suffix': self.compose_type_suffix,
|
||||
- 'label': self.compose_label,
|
||||
- 'label_major_version': self.compose_label_major_version,
|
||||
- }
|
||||
- substs.update(kwargs)
|
||||
- return substs
|
||||
-
|
||||
def get_image_name(self, arch, variant, disc_type='dvd',
|
||||
disc_num=1, suffix='.iso', format=None):
|
||||
"""Create a filename for image with given parameters.
|
||||
@@ -310,11 +291,12 @@ class Compose(kobo.log.LoggingBase):
|
||||
variant_uid = variant.parent.uid
|
||||
else:
|
||||
variant_uid = variant.uid
|
||||
- args = self.get_format_substs(variant=variant_uid,
|
||||
- arch=arch,
|
||||
- disc_type=disc_type,
|
||||
- disc_num=disc_num,
|
||||
- suffix=suffix)
|
||||
+ args = get_format_substs(self,
|
||||
+ variant=variant_uid,
|
||||
+ arch=arch,
|
||||
+ disc_type=disc_type,
|
||||
+ disc_num=disc_num,
|
||||
+ suffix=suffix)
|
||||
try:
|
||||
return format % args
|
||||
except KeyError as err:
|
||||
diff --git a/pungi/phases/image_checksum.py b/pungi/phases/image_checksum.py
|
||||
index c9ef321..aa74c4f 100644
|
||||
--- a/pungi/phases/image_checksum.py
|
||||
+++ b/pungi/phases/image_checksum.py
|
||||
@@ -4,6 +4,7 @@ import os
|
||||
from kobo import shortcuts
|
||||
|
||||
from .base import PhaseBase
|
||||
+from ..util import get_format_substs
|
||||
|
||||
|
||||
MULTIPLE_CHECKSUMS_ERROR = (
|
||||
@@ -66,29 +67,21 @@ class ImageChecksumPhase(PhaseBase):
|
||||
for arch in self.compose.im.images[variant]:
|
||||
for image in self.compose.im.images[variant][arch]:
|
||||
path = os.path.dirname(os.path.join(top_dir, image.path))
|
||||
- images.setdefault((variant, path), set()).add(image)
|
||||
+ images.setdefault((variant, arch, path), set()).add(image)
|
||||
return images
|
||||
|
||||
- def _get_base_filename(self, variant):
|
||||
+ def _get_base_filename(self, variant, arch):
|
||||
base_checksum_name = self.compose.conf.get('media_checksum_base_filename', '')
|
||||
if base_checksum_name:
|
||||
- base_checksum_name = base_checksum_name % {
|
||||
- 'release_short': self.compose.ci_base.release.short,
|
||||
- 'release_id': self.compose.ci_base.release_id,
|
||||
- 'variant': variant,
|
||||
- 'version': self.compose.ci_base.release.version,
|
||||
- 'date': self.compose.compose_date,
|
||||
- 'type_suffix': self.compose.compose_type_suffix,
|
||||
- 'respin': self.compose.compose_respin,
|
||||
- 'label': self.compose.compose_label,
|
||||
- }
|
||||
+ substs = get_format_substs(self.compose, variant=variant, arch=arch)
|
||||
+ base_checksum_name = base_checksum_name % substs
|
||||
base_checksum_name += '-'
|
||||
return base_checksum_name
|
||||
|
||||
def run(self):
|
||||
- for (variant, path), images in self._get_images().iteritems():
|
||||
+ for (variant, arch, path), images in self._get_images().iteritems():
|
||||
checksums = {}
|
||||
- base_checksum_name = self._get_base_filename(variant)
|
||||
+ base_checksum_name = self._get_base_filename(variant, arch)
|
||||
for image in images:
|
||||
filename = os.path.basename(image.path)
|
||||
full_path = os.path.join(path, filename)
|
||||
diff --git a/pungi/util.py b/pungi/util.py
|
||||
index f4f5e33..6a6ccf3 100644
|
||||
--- a/pungi/util.py
|
||||
+++ b/pungi/util.py
|
||||
@@ -370,13 +370,14 @@ def get_volid(compose, arch, variant=None, escape_spaces=False, disc_type=False)
|
||||
if not variant_uid and "%(variant)s" in i:
|
||||
continue
|
||||
try:
|
||||
- volid = i % compose.get_format_substs(variant=variant_uid,
|
||||
- release_short=release_short,
|
||||
- version=release_version,
|
||||
- arch=arch,
|
||||
- disc_type=disc_type or '',
|
||||
- base_product_short=base_product_short,
|
||||
- base_product_version=base_product_version)
|
||||
+ volid = i % get_format_substs(compose,
|
||||
+ variant=variant_uid,
|
||||
+ release_short=release_short,
|
||||
+ version=release_version,
|
||||
+ arch=arch,
|
||||
+ disc_type=disc_type or '',
|
||||
+ base_product_short=base_product_short,
|
||||
+ base_product_version=base_product_version)
|
||||
except KeyError as err:
|
||||
raise RuntimeError('Failed to create volume id: unknown format element: %s' % err.message)
|
||||
volid = _apply_substitutions(compose, volid)
|
||||
@@ -472,3 +473,23 @@ def failable(compose, variant, arch, deliverable, msg=None):
|
||||
compose.log_info(str(exc))
|
||||
tb = traceback.format_exc()
|
||||
compose.log_debug(tb)
|
||||
+
|
||||
+
|
||||
+def get_format_substs(compose, **kwargs):
|
||||
+ """Return a dict of basic format substitutions.
|
||||
+
|
||||
+ Any kwargs will be added as well.
|
||||
+ """
|
||||
+ substs = {
|
||||
+ 'compose_id': compose.compose_id,
|
||||
+ 'release_short': compose.ci_base.release.short,
|
||||
+ 'version': compose.ci_base.release.version,
|
||||
+ 'date': compose.compose_date,
|
||||
+ 'respin': compose.compose_respin,
|
||||
+ 'type': compose.compose_type,
|
||||
+ 'type_suffix': compose.compose_type_suffix,
|
||||
+ 'label': compose.compose_label,
|
||||
+ 'label_major_version': compose.compose_label_major_version,
|
||||
+ }
|
||||
+ substs.update(kwargs)
|
||||
+ return substs
|
||||
diff --git a/tests/helpers.py b/tests/helpers.py
|
||||
index 9a0983c..11c07a4 100644
|
||||
--- a/tests/helpers.py
|
||||
+++ b/tests/helpers.py
|
||||
@@ -23,9 +23,11 @@ class DummyCompose(object):
|
||||
self.supported = True
|
||||
self.compose_date = '20151203'
|
||||
self.compose_type_suffix = '.t'
|
||||
+ self.compose_type = 'test'
|
||||
self.compose_respin = 0
|
||||
self.compose_id = 'Test-20151203.0.t'
|
||||
self.compose_label = None
|
||||
+ self.compose_label_major_version = None
|
||||
self.image_release = '20151203.t.0'
|
||||
self.ci_base = mock.Mock(
|
||||
release_id='Test-1.0',
|
||||
--
|
||||
2.7.3
|
||||
|
||||
@ -1,42 +0,0 @@
|
||||
From f18e32c5affad9bf376103a536c7ded19366d92f Mon Sep 17 00:00:00 2001
|
||||
From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
|
||||
Date: Mon, 4 Apr 2016 14:24:18 +0200
|
||||
Subject: [PATCH 2/6] [atomic] Stop creating the os directory
|
||||
MIME-Version: 1.0
|
||||
Content-Type: text/plain; charset=UTF-8
|
||||
Content-Transfer-Encoding: 8bit
|
||||
|
||||
The output directory for Lorax can not exist beforehand, or the process
|
||||
will fail.
|
||||
|
||||
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
|
||||
---
|
||||
pungi/phases/atomic_installer.py | 5 +++--
|
||||
1 file changed, 3 insertions(+), 2 deletions(-)
|
||||
|
||||
diff --git a/pungi/phases/atomic_installer.py b/pungi/phases/atomic_installer.py
|
||||
index 476ad1c..3390a74 100644
|
||||
--- a/pungi/phases/atomic_installer.py
|
||||
+++ b/pungi/phases/atomic_installer.py
|
||||
@@ -49,7 +49,8 @@ class AtomicInstallerThread(WorkerThread):
|
||||
self.logdir = compose.paths.log.topdir('{}/atomic'.format(arch))
|
||||
|
||||
source_variant = compose.variants[config['source_repo_from']]
|
||||
- source_repo = translate_path(compose, compose.paths.compose.repository(arch, source_variant))
|
||||
+ source_repo = translate_path(
|
||||
+ compose, compose.paths.compose.repository(arch, source_variant, create_dir=False))
|
||||
|
||||
self._run_atomic_cmd(compose, variant, arch, config, source_repo)
|
||||
|
||||
@@ -100,7 +101,7 @@ class AtomicInstallerThread(WorkerThread):
|
||||
compose.im.add(variant.uid, arch, img)
|
||||
|
||||
def _run_atomic_cmd(self, compose, variant, arch, config, source_repo):
|
||||
- image_dir = compose.paths.compose.os_tree(arch, variant)
|
||||
+ image_dir = compose.paths.compose.os_tree(arch, variant, create_dir=False)
|
||||
lorax_wrapper = lorax.LoraxWrapper()
|
||||
cmd = lorax_wrapper.get_lorax_cmd(
|
||||
compose.conf['release_name'],
|
||||
--
|
||||
2.7.3
|
||||
|
||||
@ -1,54 +0,0 @@
|
||||
From 05384eae584476bfca3bd4bf31958d1e5dbb20a6 Mon Sep 17 00:00:00 2001
|
||||
From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
|
||||
Date: Mon, 4 Apr 2016 15:13:35 +0200
|
||||
Subject: [PATCH 3/6] [ostree] Fix call to kobo.shortcuts.run
|
||||
MIME-Version: 1.0
|
||||
Content-Type: text/plain; charset=UTF-8
|
||||
Content-Transfer-Encoding: 8bit
|
||||
|
||||
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
|
||||
---
|
||||
pungi/atomic.py | 4 ++--
|
||||
tests/test_atomic_script.py | 4 ++--
|
||||
2 files changed, 4 insertions(+), 4 deletions(-)
|
||||
|
||||
diff --git a/pungi/atomic.py b/pungi/atomic.py
|
||||
index 95ccfe0..7c0075b 100644
|
||||
--- a/pungi/atomic.py
|
||||
+++ b/pungi/atomic.py
|
||||
@@ -22,13 +22,13 @@ def init_atomic_repo(repo, log_dir=None):
|
||||
log_file = make_log_file(log_dir, 'init-atomic-repo')
|
||||
if not os.path.isdir(repo):
|
||||
shortcuts.run(['ostree', 'init', '--repo={}'.format(repo), '--mode=archive-z2'],
|
||||
- log_file=log_file)
|
||||
+ logfile=log_file)
|
||||
|
||||
|
||||
def make_ostree_repo(repo, config, log_dir=None):
|
||||
log_file = make_log_file(log_dir, 'create-atomic-repo')
|
||||
shortcuts.run(['rpm-ostree', 'compose', 'tree', '--repo={}'.format(repo), config],
|
||||
- log_file=log_file)
|
||||
+ logfile=log_file)
|
||||
|
||||
|
||||
def run(opts):
|
||||
diff --git a/tests/test_atomic_script.py b/tests/test_atomic_script.py
|
||||
index 13aa825..a871659 100755
|
||||
--- a/tests/test_atomic_script.py
|
||||
+++ b/tests/test_atomic_script.py
|
||||
@@ -28,10 +28,10 @@ class OstreeScriptTest(helpers.PungiTestCase):
|
||||
self.assertItemsEqual(
|
||||
run.call_args_list,
|
||||
[mock.call(['ostree', 'init', '--repo={}/atomic'.format(self.topdir), '--mode=archive-z2'],
|
||||
- log_file=self.topdir + '/logs/Atomic/init-atomic-repo.log'),
|
||||
+ logfile=self.topdir + '/logs/Atomic/init-atomic-repo.log'),
|
||||
mock.call(['rpm-ostree', 'compose', 'tree', '--repo={}/atomic'.format(self.topdir),
|
||||
self.topdir + '/work/fedora-atomic-docker-host.json'],
|
||||
- log_file=self.topdir + '/logs/Atomic/create-atomic-repo.log')])
|
||||
+ logfile=self.topdir + '/logs/Atomic/create-atomic-repo.log')])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
--
|
||||
2.7.3
|
||||
|
||||
@ -1,276 +0,0 @@
|
||||
From 446d4b20cc508ae466a1c5bf9cc76f8b0fc12494 Mon Sep 17 00:00:00 2001
|
||||
From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
|
||||
Date: Tue, 5 Apr 2016 08:40:54 +0200
|
||||
Subject: [PATCH 4/6] [ostree] Move cloning config repo to chroot
|
||||
MIME-Version: 1.0
|
||||
Content-Type: text/plain; charset=UTF-8
|
||||
Content-Transfer-Encoding: 8bit
|
||||
|
||||
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
|
||||
---
|
||||
pungi/atomic.py | 46 ++++++++++++++++++++++++++++++++++++++++++++-
|
||||
pungi/phases/ostree.py | 29 ++++------------------------
|
||||
tests/helpers.py | 7 ++++++-
|
||||
tests/test_atomic_script.py | 23 ++++++++++++++++++++---
|
||||
tests/test_ostree_phase.py | 15 ++++-----------
|
||||
5 files changed, 79 insertions(+), 41 deletions(-)
|
||||
|
||||
diff --git a/pungi/atomic.py b/pungi/atomic.py
|
||||
index 7c0075b..627ce34 100644
|
||||
--- a/pungi/atomic.py
|
||||
+++ b/pungi/atomic.py
|
||||
@@ -8,6 +8,11 @@ It is expected to be runnable in Koji runroot.
|
||||
import argparse
|
||||
import os
|
||||
from kobo import shortcuts
|
||||
+import tempfile
|
||||
+import shutil
|
||||
+import re
|
||||
+
|
||||
+from .wrappers import scm
|
||||
|
||||
|
||||
def make_log_file(log_dir, filename):
|
||||
@@ -31,9 +36,42 @@ def make_ostree_repo(repo, config, log_dir=None):
|
||||
logfile=log_file)
|
||||
|
||||
|
||||
+def clone_repo(repodir, url, branch):
|
||||
+ scm.get_dir_from_scm(
|
||||
+ {'scm': 'git', 'repo': url, 'branch': branch, 'dir': '.'}, repodir)
|
||||
+
|
||||
+
|
||||
+def tweak_mirrorlist(repodir, source_repo):
|
||||
+ for file in os.listdir(repodir):
|
||||
+ if file.endswith('.repo'):
|
||||
+ tweak_file(os.path.join(repodir, file), source_repo)
|
||||
+
|
||||
+
|
||||
+def tweak_file(path, source_repo):
|
||||
+ """Replace mirrorlist line in repo file with baseurl pointing to source_repo."""
|
||||
+ with open(path, 'r') as f:
|
||||
+ contents = f.read()
|
||||
+ replacement = 'baseurl={}'.format(source_repo)
|
||||
+ contents = re.sub(r'^mirrorlist=.*$', replacement, contents)
|
||||
+ with open(path, 'w') as f:
|
||||
+ f.write(contents)
|
||||
+
|
||||
+
|
||||
+def prepare_config(workdir, config_url, config_branch, source_repo):
|
||||
+ repodir = os.path.join(workdir, 'config_repo')
|
||||
+ clone_repo(repodir, config_url, config_branch)
|
||||
+ tweak_mirrorlist(repodir, source_repo)
|
||||
+ return repodir
|
||||
+
|
||||
+
|
||||
def run(opts):
|
||||
+ workdir = tempfile.mkdtemp()
|
||||
+ repodir = prepare_config(workdir, opts.config_url, opts.config_branch,
|
||||
+ opts.source_repo)
|
||||
init_atomic_repo(opts.atomic_repo, log_dir=opts.log_dir)
|
||||
- make_ostree_repo(opts.atomic_repo, opts.treefile, log_dir=opts.log_dir)
|
||||
+ treefile = os.path.join(repodir, opts.treefile)
|
||||
+ make_ostree_repo(opts.atomic_repo, treefile, log_dir=opts.log_dir)
|
||||
+ shutil.rmtree(workdir)
|
||||
|
||||
|
||||
def main(args=None):
|
||||
@@ -45,6 +83,12 @@ def main(args=None):
|
||||
help='where to put the atomic repo')
|
||||
parser.add_argument('--treefile', required=True,
|
||||
help='treefile for rpm-ostree')
|
||||
+ parser.add_argument('--config-url', required=True,
|
||||
+ help='git repository with the treefile')
|
||||
+ parser.add_argument('--config-branch', default='master',
|
||||
+ help='git branch to be used')
|
||||
+ parser.add_argument('--source-repo', required=True,
|
||||
+ help='yum repo used as source for')
|
||||
|
||||
opts = parser.parse_args(args)
|
||||
|
||||
diff --git a/pungi/phases/ostree.py b/pungi/phases/ostree.py
|
||||
index 36cbc97..79e58d4 100644
|
||||
--- a/pungi/phases/ostree.py
|
||||
+++ b/pungi/phases/ostree.py
|
||||
@@ -2,12 +2,11 @@
|
||||
|
||||
import os
|
||||
from kobo.threads import ThreadPool, WorkerThread
|
||||
-import re
|
||||
|
||||
from .base import ConfigGuardedPhase
|
||||
from .. import util
|
||||
from ..paths import translate_path
|
||||
-from ..wrappers import scm, kojiwrapper
|
||||
+from ..wrappers import kojiwrapper
|
||||
|
||||
|
||||
class OSTreePhase(ConfigGuardedPhase):
|
||||
@@ -42,15 +41,11 @@ class OSTreeThread(WorkerThread):
|
||||
|
||||
msg = 'OSTree phase for variant %s, arch %s' % (variant.uid, arch)
|
||||
self.pool.log_info('[BEGIN] %s' % msg)
|
||||
- workdir = compose.paths.work.topdir('atomic')
|
||||
self.logdir = compose.paths.log.topdir('{}/atomic'.format(arch))
|
||||
- repodir = os.path.join(workdir, 'config_repo')
|
||||
|
||||
source_variant = compose.variants[config['source_repo_from']]
|
||||
source_repo = translate_path(compose, compose.paths.compose.repository(arch, source_variant))
|
||||
|
||||
- self._clone_repo(repodir, config['config_url'], config.get('config_branch', 'master'))
|
||||
- self._tweak_mirrorlist(repodir, source_repo)
|
||||
self._run_atomic_cmd(compose, variant, arch, config, source_repo)
|
||||
|
||||
self.pool.log_info('[DONE ] %s' % msg)
|
||||
@@ -60,6 +55,9 @@ class OSTreeThread(WorkerThread):
|
||||
'pungi-make-ostree',
|
||||
'--log-dir={}'.format(self.logdir),
|
||||
'--treefile={}'.format(config['treefile']),
|
||||
+ '--config-url={}'.format(config['config_url']),
|
||||
+ '--config-branch={}'.format(config.get('config_branch', 'master')),
|
||||
+ '--source-repo={}'.format(source_repo),
|
||||
config['atomic_repo']
|
||||
]
|
||||
|
||||
@@ -77,22 +75,3 @@ class OSTreeThread(WorkerThread):
|
||||
if output["retcode"] != 0:
|
||||
raise RuntimeError("Runroot task failed: %s. See %s for more details."
|
||||
% (output["task_id"], log_file))
|
||||
-
|
||||
- def _clone_repo(self, repodir, url, branch):
|
||||
- scm.get_dir_from_scm({'scm': 'git', 'repo': url, 'branch': branch, 'dir': '.'},
|
||||
- repodir, logger=self.pool._logger)
|
||||
-
|
||||
- def _tweak_mirrorlist(self, repodir, source_repo):
|
||||
- for file in os.listdir(repodir):
|
||||
- if file.endswith('.repo'):
|
||||
- tweak_file(os.path.join(repodir, file), source_repo)
|
||||
-
|
||||
-
|
||||
-def tweak_file(path, source_repo):
|
||||
- """Replace mirrorlist line in repo file with baseurl pointing to source_repo."""
|
||||
- with open(path, 'r') as f:
|
||||
- contents = f.read()
|
||||
- replacement = 'baseurl={}'.format(source_repo)
|
||||
- contents = re.sub(r'^mirrorlist=.*$', replacement, contents)
|
||||
- with open(path, 'w') as f:
|
||||
- f.write(contents)
|
||||
diff --git a/tests/helpers.py b/tests/helpers.py
|
||||
index 11c07a4..d93e095 100644
|
||||
--- a/tests/helpers.py
|
||||
+++ b/tests/helpers.py
|
||||
@@ -5,6 +5,7 @@ import os
|
||||
import unittest
|
||||
import tempfile
|
||||
import shutil
|
||||
+import errno
|
||||
|
||||
from pungi.util import get_arch_variant_data
|
||||
from pungi import paths
|
||||
@@ -15,7 +16,11 @@ class PungiTestCase(unittest.TestCase):
|
||||
self.topdir = tempfile.mkdtemp()
|
||||
|
||||
def tearDown(self):
|
||||
- shutil.rmtree(self.topdir)
|
||||
+ try:
|
||||
+ shutil.rmtree(self.topdir)
|
||||
+ except OSError as err:
|
||||
+ if err.errno != errno.ENOENT:
|
||||
+ raise
|
||||
|
||||
|
||||
class DummyCompose(object):
|
||||
diff --git a/tests/test_atomic_script.py b/tests/test_atomic_script.py
|
||||
index a871659..4b0b396 100755
|
||||
--- a/tests/test_atomic_script.py
|
||||
+++ b/tests/test_atomic_script.py
|
||||
@@ -17,20 +17,37 @@ from pungi import atomic
|
||||
|
||||
class OstreeScriptTest(helpers.PungiTestCase):
|
||||
|
||||
+ def _dummy_config_repo(self, scm_dict, target, logger=None):
|
||||
+ helpers.touch(os.path.join(target, 'fedora-atomic-docker-host.json'))
|
||||
+ helpers.touch(os.path.join(target, 'fedora-rawhide.repo'))
|
||||
+
|
||||
+ @mock.patch('tempfile.mkdtemp')
|
||||
@mock.patch('kobo.shortcuts.run')
|
||||
- def test_full_run(self, run):
|
||||
+ @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
|
||||
+ def test_full_run(self, get_dir_from_scm, run, tempfile):
|
||||
+ tempfile.return_value = self.topdir
|
||||
+ get_dir_from_scm.side_effect = self._dummy_config_repo
|
||||
+
|
||||
atomic.main([
|
||||
'--log-dir={}'.format(os.path.join(self.topdir, 'logs', 'Atomic')),
|
||||
- '--treefile={}'.format(os.path.join(self.topdir, 'work', 'fedora-atomic-docker-host.json')),
|
||||
+ '--treefile={}'.format('fedora-atomic-docker-host.json'),
|
||||
+ '--config-url=https://git.fedorahosted.org/git/fedora-atomic.git',
|
||||
+ '--config-branch=f24',
|
||||
+ '--source-repo=https://kojipkgs.fedoraproject.org/repo',
|
||||
os.path.join(self.topdir, 'atomic'),
|
||||
])
|
||||
+
|
||||
self.maxDiff = None
|
||||
+ self.assertEqual(get_dir_from_scm.call_args_list,
|
||||
+ [mock.call({'scm': 'git', 'repo': 'https://git.fedorahosted.org/git/fedora-atomic.git',
|
||||
+ 'branch': 'f24', 'dir': '.'},
|
||||
+ self.topdir + '/config_repo')])
|
||||
self.assertItemsEqual(
|
||||
run.call_args_list,
|
||||
[mock.call(['ostree', 'init', '--repo={}/atomic'.format(self.topdir), '--mode=archive-z2'],
|
||||
logfile=self.topdir + '/logs/Atomic/init-atomic-repo.log'),
|
||||
mock.call(['rpm-ostree', 'compose', 'tree', '--repo={}/atomic'.format(self.topdir),
|
||||
- self.topdir + '/work/fedora-atomic-docker-host.json'],
|
||||
+ self.topdir + '/config_repo/fedora-atomic-docker-host.json'],
|
||||
logfile=self.topdir + '/logs/Atomic/create-atomic-repo.log')])
|
||||
|
||||
|
||||
diff --git a/tests/test_ostree_phase.py b/tests/test_ostree_phase.py
|
||||
index e4664e1..fe0036c 100755
|
||||
--- a/tests/test_ostree_phase.py
|
||||
+++ b/tests/test_ostree_phase.py
|
||||
@@ -74,13 +74,8 @@ class OSTreePhaseTest(helpers.PungiTestCase):
|
||||
|
||||
class OSTreeThreadTest(helpers.PungiTestCase):
|
||||
|
||||
- def _dummy_config_repo(self, scm_dict, target, logger=None):
|
||||
- helpers.touch(os.path.join(target, 'fedora-atomic-docker-host.json'))
|
||||
- helpers.touch(os.path.join(target, 'fedora-rawhide.repo'))
|
||||
-
|
||||
- @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
|
||||
@mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
|
||||
- def test_run(self, KojiWrapper, get_dir_from_scm):
|
||||
+ def test_run(self, KojiWrapper):
|
||||
compose = helpers.DummyCompose(self.topdir, {
|
||||
'koji_profile': 'koji',
|
||||
'runroot_tag': 'rrt',
|
||||
@@ -93,7 +88,6 @@ class OSTreeThreadTest(helpers.PungiTestCase):
|
||||
'treefile': 'fedora-atomic-docker-host.json',
|
||||
'atomic_repo': '/other/place/for/atomic'
|
||||
}
|
||||
- get_dir_from_scm.side_effect = self._dummy_config_repo
|
||||
koji = KojiWrapper.return_value
|
||||
koji.run_runroot_cmd.return_value = {
|
||||
'task_id': 1234,
|
||||
@@ -105,15 +99,14 @@ class OSTreeThreadTest(helpers.PungiTestCase):
|
||||
|
||||
t.process((compose, compose.variants['Everything'], 'x86_64', cfg), 1)
|
||||
|
||||
- self.assertEqual(get_dir_from_scm.call_args_list,
|
||||
- [mock.call({'scm': 'git', 'repo': 'https://git.fedorahosted.org/git/fedora-atomic.git',
|
||||
- 'branch': 'f24', 'dir': '.'},
|
||||
- self.topdir + '/work/atomic/config_repo', logger=pool._logger)])
|
||||
self.assertEqual(koji.get_runroot_cmd.call_args_list,
|
||||
[mock.call('rrt', 'x86_64',
|
||||
['pungi-make-ostree',
|
||||
'--log-dir={}/logs/x86_64/atomic'.format(self.topdir),
|
||||
'--treefile=fedora-atomic-docker-host.json',
|
||||
+ '--config-url=https://git.fedorahosted.org/git/fedora-atomic.git',
|
||||
+ '--config-branch=f24',
|
||||
+ '--source-repo={}/compose/Everything/x86_64/os'.format(self.topdir),
|
||||
'/other/place/for/atomic'],
|
||||
channel=None, mounts=[self.topdir],
|
||||
packages=['pungi', 'ostree', 'rpm-ostree'],
|
||||
--
|
||||
2.7.3
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,168 +0,0 @@
|
||||
From 79ee9a4ffaf4627eade703c9bd819c870258745f Mon Sep 17 00:00:00 2001
|
||||
From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
|
||||
Date: Tue, 5 Apr 2016 12:46:59 +0200
|
||||
Subject: [PATCH 6/6] [ostree] Use explicit work directory
|
||||
MIME-Version: 1.0
|
||||
Content-Type: text/plain; charset=UTF-8
|
||||
Content-Transfer-Encoding: 8bit
|
||||
|
||||
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
|
||||
---
|
||||
pungi/ostree.py | 26 +++++++++++++++++++-------
|
||||
pungi/phases/ostree.py | 2 ++
|
||||
tests/test_ostree_phase.py | 1 +
|
||||
tests/test_ostree_script.py | 9 ++++-----
|
||||
4 files changed, 26 insertions(+), 12 deletions(-)
|
||||
|
||||
diff --git a/pungi/ostree.py b/pungi/ostree.py
|
||||
index de2cfcd..c0c8a04 100644
|
||||
--- a/pungi/ostree.py
|
||||
+++ b/pungi/ostree.py
|
||||
@@ -8,17 +8,26 @@ It is expected to be runnable in Koji runroot.
|
||||
import argparse
|
||||
import os
|
||||
from kobo import shortcuts
|
||||
-import tempfile
|
||||
-import shutil
|
||||
import re
|
||||
+import errno
|
||||
|
||||
from .wrappers import scm
|
||||
|
||||
|
||||
+def ensure_dir(path):
|
||||
+ try:
|
||||
+ os.makedirs(path)
|
||||
+ except OSError as err:
|
||||
+ if err.errno != errno.EEXIST:
|
||||
+ raise
|
||||
+ return path
|
||||
+
|
||||
+
|
||||
def make_log_file(log_dir, filename):
|
||||
"""Return path to log file with given name, if log_dir is set."""
|
||||
if not log_dir:
|
||||
return None
|
||||
+ ensure_dir(log_dir)
|
||||
return os.path.join(log_dir, '{}.log'.format(filename))
|
||||
|
||||
|
||||
@@ -26,14 +35,15 @@ def init_ostree_repo(repo, log_dir=None):
|
||||
"""If the ostree repo does not exist, initialize it."""
|
||||
log_file = make_log_file(log_dir, 'init-ostree-repo')
|
||||
if not os.path.isdir(repo):
|
||||
+ ensure_dir(repo)
|
||||
shortcuts.run(['ostree', 'init', '--repo={}'.format(repo), '--mode=archive-z2'],
|
||||
- logfile=log_file)
|
||||
+ show_cmd=True, logfile=log_file)
|
||||
|
||||
|
||||
def make_ostree_repo(repo, config, log_dir=None):
|
||||
log_file = make_log_file(log_dir, 'create-ostree-repo')
|
||||
shortcuts.run(['rpm-ostree', 'compose', 'tree', '--repo={}'.format(repo), config],
|
||||
- logfile=log_file)
|
||||
+ show_cmd=True, logfile=log_file)
|
||||
|
||||
|
||||
def clone_repo(repodir, url, branch):
|
||||
@@ -52,7 +62,7 @@ def tweak_file(path, source_repo):
|
||||
with open(path, 'r') as f:
|
||||
contents = f.read()
|
||||
replacement = 'baseurl={}'.format(source_repo)
|
||||
- contents = re.sub(r'^mirrorlist=.*$', replacement, contents)
|
||||
+ contents = re.sub(r'^mirrorlist=.*$', replacement, contents, flags=re.MULTILINE)
|
||||
with open(path, 'w') as f:
|
||||
f.write(contents)
|
||||
|
||||
@@ -65,19 +75,21 @@ def prepare_config(workdir, config_url, config_branch, source_repo):
|
||||
|
||||
|
||||
def run(opts):
|
||||
- workdir = tempfile.mkdtemp()
|
||||
+ workdir = ensure_dir(opts.work_dir)
|
||||
repodir = prepare_config(workdir, opts.config_url, opts.config_branch,
|
||||
opts.source_repo)
|
||||
+ ensure_dir(repodir)
|
||||
init_ostree_repo(opts.ostree_repo, log_dir=opts.log_dir)
|
||||
treefile = os.path.join(repodir, opts.treefile)
|
||||
make_ostree_repo(opts.ostree_repo, treefile, log_dir=opts.log_dir)
|
||||
- shutil.rmtree(workdir)
|
||||
|
||||
|
||||
def main(args=None):
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--log-dir',
|
||||
help='where to log output')
|
||||
+ parser.add_argument('--work-dir', required=True,
|
||||
+ help='where to put temporary files')
|
||||
|
||||
parser.add_argument('ostree_repo', metavar='OSTREE_REPO',
|
||||
help='where to put the ostree repo')
|
||||
diff --git a/pungi/phases/ostree.py b/pungi/phases/ostree.py
|
||||
index 810716e..77e4838 100644
|
||||
--- a/pungi/phases/ostree.py
|
||||
+++ b/pungi/phases/ostree.py
|
||||
@@ -51,9 +51,11 @@ class OSTreeThread(WorkerThread):
|
||||
self.pool.log_info('[DONE ] %s' % msg)
|
||||
|
||||
def _run_ostree_cmd(self, compose, variant, arch, config, source_repo):
|
||||
+ workdir = os.path.join(compose.paths.work.topdir(arch), 'ostree')
|
||||
cmd = [
|
||||
'pungi-make-ostree',
|
||||
'--log-dir={}'.format(self.logdir),
|
||||
+ '--work-dir={}'.format(workdir),
|
||||
'--treefile={}'.format(config['treefile']),
|
||||
'--config-url={}'.format(config['config_url']),
|
||||
'--config-branch={}'.format(config.get('config_branch', 'master')),
|
||||
diff --git a/tests/test_ostree_phase.py b/tests/test_ostree_phase.py
|
||||
index 9f8fdf1..7617dde 100755
|
||||
--- a/tests/test_ostree_phase.py
|
||||
+++ b/tests/test_ostree_phase.py
|
||||
@@ -103,6 +103,7 @@ class OSTreeThreadTest(helpers.PungiTestCase):
|
||||
[mock.call('rrt', 'x86_64',
|
||||
['pungi-make-ostree',
|
||||
'--log-dir={}/logs/x86_64/ostree'.format(self.topdir),
|
||||
+ '--work-dir={}/work/x86_64/ostree'.format(self.topdir),
|
||||
'--treefile=fedora-atomic-docker-host.json',
|
||||
'--config-url=https://git.fedorahosted.org/git/fedora-atomic.git',
|
||||
'--config-branch=f24',
|
||||
diff --git a/tests/test_ostree_script.py b/tests/test_ostree_script.py
|
||||
index 744af4d..7b59680 100755
|
||||
--- a/tests/test_ostree_script.py
|
||||
+++ b/tests/test_ostree_script.py
|
||||
@@ -21,17 +21,16 @@ class OstreeScriptTest(helpers.PungiTestCase):
|
||||
helpers.touch(os.path.join(target, 'fedora-atomic-docker-host.json'))
|
||||
helpers.touch(os.path.join(target, 'fedora-rawhide.repo'))
|
||||
|
||||
- @mock.patch('tempfile.mkdtemp')
|
||||
@mock.patch('kobo.shortcuts.run')
|
||||
@mock.patch('pungi.wrappers.scm.get_dir_from_scm')
|
||||
- def test_full_run(self, get_dir_from_scm, run, tempfile):
|
||||
- tempfile.return_value = self.topdir
|
||||
+ def test_full_run(self, get_dir_from_scm, run):
|
||||
get_dir_from_scm.side_effect = self._dummy_config_repo
|
||||
|
||||
repo = os.path.join(self.topdir, 'atomic')
|
||||
|
||||
ostree.main([
|
||||
'--log-dir={}'.format(os.path.join(self.topdir, 'logs', 'Atomic')),
|
||||
+ '--work-dir={}'.format(self.topdir),
|
||||
'--treefile={}'.format('fedora-atomic-docker-host.json'),
|
||||
'--config-url=https://git.fedorahosted.org/git/fedora-atomic.git',
|
||||
'--config-branch=f24',
|
||||
@@ -47,10 +46,10 @@ class OstreeScriptTest(helpers.PungiTestCase):
|
||||
self.assertItemsEqual(
|
||||
run.call_args_list,
|
||||
[mock.call(['ostree', 'init', '--repo={}'.format(repo), '--mode=archive-z2'],
|
||||
- logfile=self.topdir + '/logs/Atomic/init-ostree-repo.log'),
|
||||
+ logfile=self.topdir + '/logs/Atomic/init-ostree-repo.log', show_cmd=True),
|
||||
mock.call(['rpm-ostree', 'compose', 'tree', '--repo={}'.format(repo),
|
||||
self.topdir + '/config_repo/fedora-atomic-docker-host.json'],
|
||||
- logfile=self.topdir + '/logs/Atomic/create-ostree-repo.log')])
|
||||
+ logfile=self.topdir + '/logs/Atomic/create-ostree-repo.log', show_cmd=True)])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
--
|
||||
2.7.3
|
||||
|
||||
@ -1,54 +0,0 @@
|
||||
From c52cb198c5ffc0593720c927f469793bedb3d4a4 Mon Sep 17 00:00:00 2001
|
||||
From: Peter Robinson <pbrobinson@gmail.com>
|
||||
Date: Mar 16 2016 22:03:33 +0000
|
||||
Subject: Fix locations of ppc lorax templates for PowerPC iso creation
|
||||
|
||||
|
||||
PowerPC needs a mapping file for iso creation, with lorax moving the
|
||||
templates locations in 24+ we need to update this so runroot iso
|
||||
creation doesn't fail.
|
||||
|
||||
Signed-off-by: Peter Robinson <pbrobinson@fedoraproject.org>
|
||||
|
||||
---
|
||||
|
||||
diff --git a/pungi/gather.py b/pungi/gather.py
|
||||
index 2e5048c..c27070f 100644
|
||||
--- a/pungi/gather.py
|
||||
+++ b/pungi/gather.py
|
||||
@@ -1590,7 +1590,7 @@ class Pungi(PungiBase):
|
||||
return
|
||||
|
||||
isolist = []
|
||||
- ppcbootinfo = '/usr/share/lorax/config_files/ppc'
|
||||
+ ppcbootinfo = '/usr/share/lorax/templates.d/99-generic/config_files/ppc'
|
||||
|
||||
pungi.util._ensuredir(self.isodir, self.logger,
|
||||
force=self.config.getboolean('pungi', 'force'),
|
||||
diff --git a/pungi/phases/createiso.py b/pungi/phases/createiso.py
|
||||
index f88b20d..eab3133 100644
|
||||
--- a/pungi/phases/createiso.py
|
||||
+++ b/pungi/phases/createiso.py
|
||||
@@ -142,7 +142,7 @@ class CreateisoPhase(PhaseBase):
|
||||
buildinstall_method = self.compose.conf["buildinstall_method"]
|
||||
if buildinstall_method == "lorax":
|
||||
# TODO: $arch instead of ppc
|
||||
- mkisofs_kwargs["boot_args"] = iso.get_boot_options(arch, "/usr/share/lorax/config_files/ppc")
|
||||
+ mkisofs_kwargs["boot_args"] = iso.get_boot_options(arch, "/usr/share/lorax/templates.d/99-generic/config_files/ppc")
|
||||
elif buildinstall_method == "buildinstall":
|
||||
mkisofs_kwargs["boot_args"] = iso.get_boot_options(arch, "/usr/lib/anaconda-runtime/boot")
|
||||
|
||||
diff --git a/pungi/phases/product_img.py b/pungi/phases/product_img.py
|
||||
index 687d261..8da4e23 100644
|
||||
--- a/pungi/phases/product_img.py
|
||||
+++ b/pungi/phases/product_img.py
|
||||
@@ -230,7 +230,7 @@ def rebuild_boot_iso(compose, arch, variant, package_sets):
|
||||
boot_files = None
|
||||
if buildinstall_method == "lorax":
|
||||
# TODO: $arch instead of ppc
|
||||
- mkisofs_kwargs["boot_args"] = iso.get_boot_options(arch, "/usr/share/lorax/config_files/ppc")
|
||||
+ mkisofs_kwargs["boot_args"] = iso.get_boot_options(arch, "/usr/share/lorax/templates.d/99-generic/config_files/ppc")
|
||||
elif buildinstall_method == "buildinstall":
|
||||
boot_files = explode_anaconda(compose, arch, variant, package_sets)
|
||||
mkisofs_kwargs["boot_args"] = iso.get_boot_options(arch, boot_files)
|
||||
|
||||
32
pungi.spec
32
pungi.spec
@ -1,20 +1,12 @@
|
||||
Name: pungi
|
||||
Version: 4.1.1
|
||||
Release: 3%{?dist}
|
||||
Version: 4.1.2
|
||||
Release: 1%{?dist}
|
||||
Summary: Distribution compose tool
|
||||
|
||||
Group: Development/Tools
|
||||
License: GPLv2
|
||||
URL: https://pagure.io/pungi
|
||||
Source0: https://fedorahosted.org/pungi/attachment/wiki/%{version}/%{name}-%{version}.tar.bz2
|
||||
Patch0: 0001-checksum-Add-arch-to-file-name.patch
|
||||
Patch1: 0002-atomic-Stop-creating-the-os-directory.patch
|
||||
Patch2: 0003-ostree-Fix-call-to-kobo.shortcuts.run.patch
|
||||
Patch3: 0004-ostree-Move-cloning-config-repo-to-chroot.patch
|
||||
Patch4: 0005-ostree-Rename-atomic-to-ostree.patch
|
||||
Patch5: 0006-ostree-Use-explicit-work-directory.patch
|
||||
# bandaid for https://pagure.io/pungi/issue/231 until we get things fixed properly upstream
|
||||
Patch6: c52cb198c5ffc0593720c927f469793bedb3d4a4.patch
|
||||
|
||||
BuildRequires: python-nose, python-nose-cov, python-mock
|
||||
BuildRequires: python-devel, python-setuptools, python2-productmd
|
||||
@ -56,13 +48,6 @@ A tool to create anaconda based installation trees/isos of a set of rpms.
|
||||
|
||||
%prep
|
||||
%setup -q
|
||||
%patch0 -p1
|
||||
%patch1 -p1
|
||||
%patch2 -p1
|
||||
%patch3 -p1
|
||||
%patch4 -p1
|
||||
%patch5 -p1
|
||||
%patch6 -p1
|
||||
|
||||
%build
|
||||
%{__python} setup.py build
|
||||
@ -96,6 +81,19 @@ cd tests && ./test_compose.sh
|
||||
/var/cache/pungi
|
||||
|
||||
%changelog
|
||||
* Wed Apr 05 2016 Dennis Gilmore <dennis@ausil.us> - 4.1.2-1
|
||||
- Merge #257 `[ostree] Enable marking ostree phase as failable` (ausil)
|
||||
- [ostree] Enable marking ostree phase as failable (lsedlar)
|
||||
- [koji-wrapper] Initialize wrappers sequentially (lsedlar)
|
||||
- [createiso] Simplify code, test phase (lsedlar)
|
||||
- [createiso] Move runroot work to separate script (lsedlar)
|
||||
- [ostree] Use explicit work directory (lsedlar)
|
||||
- [ostree] Rename atomic to ostree (lsedlar)
|
||||
- [ostree] Move cloning config repo to chroot (lsedlar)
|
||||
- [ostree] Fix call to kobo.shortcuts.run (lsedlar)
|
||||
- [atomic] Stop creating the os directory (lsedlar)
|
||||
- [checksum] Add arch to file name (lsedlar)
|
||||
|
||||
* Tue Apr 05 2016 Dennis Gilmore <dennis@ausil.us> - 4.1.1-3
|
||||
- add some more ostree fixes
|
||||
- add a bandaid for ppc until we get a proper fix
|
||||
|
||||
Loading…
Reference in New Issue
Block a user