almalinux
/
pungi
15
1
Fork 0
Code Issues Pull Requests Releases Activity

Add support for module defaults

This commit is contained in:
Lubomír Sedlář 2018-04-12 11:04:22 +02:00
parent 12c29eb4c5
commit 6c327d6971
9 changed files with 311 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
0001-Revert-Move-ostree-phase-and-pipelines-for-running-p.patch
View File

@ -1,7 +1,7 @@
From 3f60e62ea86a3180b45290102d3519347cf8788a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
Date: Wed, 11 Apr 2018 09:18:59 +0200
Subject: [PATCH 1/3] Revert "Move ostree phase and pipelines for running
Subject: [PATCH 1/8] Revert "Move ostree phase and pipelines for running
phases"
This reverts commit 660c04368ba1abed310f121d01f0fa029eea5f11.

2
0002-Revert-Other-repo-for-OstreeInstaller.patch
View File

@ -1,7 +1,7 @@
From fcc431622fc0305248bb966271c71c8d17fd5f69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
Date: Wed, 11 Apr 2018 09:19:53 +0200
Subject: [PATCH 2/3] Revert "Other repo for OstreeInstaller"
Subject: [PATCH 2/8] Revert "Other repo for OstreeInstaller"
This reverts commit 5c081cb545715c2a912ff50fa57554e89d905868.
---

2
0003-Revert-Ostree-can-use-pkgset-repos.patch
View File

@ -1,7 +1,7 @@
From 8a7bf97434cbbf2397d3209498eacc326fc130f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
Date: Wed, 11 Apr 2018 09:20:51 +0200
Subject: [PATCH 3/3] Revert "Ostree can use pkgset repos"
Subject: [PATCH 3/8] Revert "Ostree can use pkgset repos"
This reverts commit c7cc200246300c6a3946b2e3a9f5f7693896a7d6.
---

131
0004-Include-module-defaults-in-the-repodata.patch Normal file
View File

@ -0,0 +1,131 @@
From 7d3baa8531e11059866412ddc6ae1aae4e855fc2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Petr=20=C5=A0abata?= <contyk@redhat.com>
Date: Wed, 4 Apr 2018 12:59:52 +0200
Subject: [PATCH 4/8] Include module defaults in the repodata
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
If the compose configuration includes the module_defaults_dir (an
scm_dict), clone the directory, read the module defaults contained
therein and include relevant defaults in the combined modulemd file.
Only defaults for modules present in the variant are included.
This requires libmodulemd 1.2.0+.
Merges: https://pagure.io/pungi/pull-request/891
Signed-off-by: Petr Šabata <contyk@redhat.com>
---
pungi/compose.py | 4 ++++
pungi/phases/createrepo.py | 8 +++++++-
pungi/phases/init.py | 17 ++++++++++++++++-
tests/test_initphase.py | 3 +++
4 files changed, 30 insertions(+), 2 deletions(-)
diff --git a/pungi/compose.py b/pungi/compose.py
index 4199d72a..07c67f4f 100644
--- a/pungi/compose.py
+++ b/pungi/compose.py
@@ -194,6 +194,10 @@ class Compose(kobo.log.LoggingBase):
return bool(self.conf.get("comps_file", False))
@property
+ def has_module_defaults(self):
+ return bool(self.conf.get("module_defaults_dir", False))
+
+ @property
def config_dir(self):
return os.path.dirname(self.conf._open_file or "")
diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py
index 1ce2b171..49ff553a 100644
--- a/pungi/phases/createrepo.py
+++ b/pungi/phases/createrepo.py
@@ -218,9 +218,15 @@ def create_variant_repo(compose, arch, variant, pkg_type, modules_metadata=None)
raise AttributeError("module_metadata parameter was not passed and it is needed for module processing")
modules.append(repo_mmd)
+ module_names = set([x.get_name() for x in modules])
+ for mmddeffile in glob.glob(os.path.join(compose.config_dir, "module_defaults", "*.yaml")):
+ for mmddef in Modulemd.objects_from_file(mmddeffile):
+ if isinstance(mmddef, Modulemd.Defaults) and mmddef.peek_module_name() in module_names:
+ modules.append(mmddef)
+
with temp_dir() as tmp_dir:
modules_path = os.path.join(tmp_dir, "modules.yaml")
- Modulemd.Module.dump_all(modules, modules_path)
+ Modulemd.dump(modules, modules_path)
cmd = repo.get_modifyrepo_cmd(os.path.join(repo_dir, "repodata"),
modules_path, mdtype="modules",
diff --git a/pungi/phases/init.py b/pungi/phases/init.py
index a01168a9..81eec0f4 100644
--- a/pungi/phases/init.py
+++ b/pungi/phases/init.py
@@ -23,7 +23,8 @@ from pungi.phases.base import PhaseBase
from pungi.phases.gather import write_prepopulate_file
from pungi.wrappers.createrepo import CreaterepoWrapper
from pungi.wrappers.comps import CompsWrapper
-from pungi.wrappers.scm import get_file_from_scm
+from pungi.wrappers.scm import get_file_from_scm, get_dir_from_scm
+from pungi.util import temp_dir
class InitPhase(PhaseBase):
@@ -63,6 +64,10 @@ class InitPhase(PhaseBase):
# download variants.xml / product.xml?
+ # download module defaults
+ if self.compose.has_module_defaults:
+ write_module_defaults(self.compose)
+
# write prepopulate file
write_prepopulate_file(self.compose)
@@ -160,3 +165,13 @@ def create_comps_repo(compose, arch):
checksum=createrepo_checksum)
run(cmd, logfile=compose.paths.log.log_file(arch, "comps_repo"), show_cmd=True)
compose.log_info("[DONE ] %s" % msg)
+
+
+def write_module_defaults(compose):
+ scm_dict = compose.conf["module_defaults_dir"]
+
+ with temp_dir(prefix="moduledefaults_") as tmp_dir:
+ get_dir_from_scm(scm_dict, tmp_dir, logger=compose._logger)
+ compose.log_debug("Writing module defaults")
+ shutil.rmtree(os.path.join(compose.config_dir, "module_defaults"), ignore_errors=True)
+ shutil.copytree(tmp_dir, os.path.join(compose.config_dir, "module_defaults"))
diff --git a/tests/test_initphase.py b/tests/test_initphase.py
index ae6c4e52..8c3b0d9d 100644
--- a/tests/test_initphase.py
+++ b/tests/test_initphase.py
@@ -24,6 +24,7 @@ class TestInitPhase(PungiTestCase):
def test_run(self, write_prepopulate, write_variant, create_comps, write_arch, write_global):
compose = DummyCompose(self.topdir, {})
compose.has_comps = True
+ compose.has_module_defaults = False
compose.setup_optional()
phase = init.InitPhase(compose)
phase.run()
@@ -52,6 +53,7 @@ class TestInitPhase(PungiTestCase):
write_arch, write_global, copy_comps):
compose = DummyCompose(self.topdir, {})
compose.has_comps = True
+ compose.has_module_defaults = False
compose.variants['Everything'].groups = []
compose.variants['Everything'].modules = []
phase = init.InitPhase(compose)
@@ -81,6 +83,7 @@ class TestInitPhase(PungiTestCase):
write_arch, write_global, copy_comps):
compose = DummyCompose(self.topdir, {})
compose.has_comps = False
+ compose.has_module_defaults = False
phase = init.InitPhase(compose)
phase.run()
--
2.13.6

35
0005-Handle-relative-paths-in-module_defaults_dir.patch Normal file
View File

@ -0,0 +1,35 @@
From 91a03c693222a41a23d1a3250984e0d8c9372b75 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Petr=20=C5=A0abata?= <contyk@redhat.com>
Date: Wed, 4 Apr 2018 14:01:10 +0200
Subject: [PATCH 5/8] Handle relative paths in module_defaults_dir
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
This now handles strings as well as sct_dicts with type=file that
include relative paths.
Signed-off-by: Petr Šabata <contyk@redhat.com>
---
pungi/phases/init.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/pungi/phases/init.py b/pungi/phases/init.py
index 81eec0f4..ef203568 100644
--- a/pungi/phases/init.py
+++ b/pungi/phases/init.py
@@ -169,6 +169,11 @@ def create_comps_repo(compose, arch):
def write_module_defaults(compose):
scm_dict = compose.conf["module_defaults_dir"]
+ if isinstance(scm_dict, dict):
+ if scm_dict["scm"] == "file":
+ scm_dict["dir"] = os.path.join(compose.config_dir, scm_dict["dir"])
+ else:
+ scm_dict = os.path.join(compose.config_dir, scm_dict)
with temp_dir(prefix="moduledefaults_") as tmp_dir:
get_dir_from_scm(scm_dict, tmp_dir, logger=compose._logger)
--
2.13.6

39
0006-Update-configuration-docs-with-module_defaults_dir.patch Normal file
View File

@ -0,0 +1,39 @@
From 587b4847843af878650d56e7c093cbef6b356801 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Petr=20=C5=A0abata?= <contyk@redhat.com>
Date: Wed, 4 Apr 2018 14:02:15 +0200
Subject: [PATCH 6/8] Update configuration docs with module_defaults_dir
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Petr Šabata <contyk@redhat.com>
---
doc/configuration.rst | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/doc/configuration.rst b/doc/configuration.rst
index e5f1e9e0..9e6cf1a9 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -21,6 +21,7 @@ Minimal Config Example
# GENERAL SETTINGS
comps_file = "comps-f23.xml"
variants_file = "variants-f23.xml"
+ module_defaults_dir = "module_defaults"
# KOJI
koji_profile = "koji"
@@ -135,6 +136,10 @@ Options
(:ref:`scm_dict <scm_support>` or *str*) -- reference to variants XML file
that defines release variants and architectures
+**module_defaults_dir** [optional]
+ (:ref:`scm_dict <scm_support>` or *str*) -- reference the module defaults
+ directory containing modulemd-defaults YAML documents
+
**failable_deliverables** [optional]
(*list*) -- list which deliverables on which variant and architecture can
fail and not abort the whole compose. This only applies to ``buildinstall``
--
2.13.6

29
0007-Update-the-configuration-JSON-schema-for-module_defa.patch Normal file
View File

@ -0,0 +1,29 @@
From dc6fccd522efc0ae0c2072812b6fc0559cc74475 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Petr=20=C5=A0abata?= <contyk@redhat.com>
Date: Wed, 11 Apr 2018 10:33:42 +0200
Subject: [PATCH 7/8] Update the configuration JSON schema for
module_defaults_dir
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Petr Šabata <contyk@redhat.com>
---
pungi/checks.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/pungi/checks.py b/pungi/checks.py
index 189ca948..1c2059b2 100644
--- a/pungi/checks.py
+++ b/pungi/checks.py
@@ -678,6 +678,7 @@ def make_schema():
"type": "boolean",
"default": True
},
+ "module_defaults_dir": {"$ref": "#/definitions/str_or_scm_dict"},
"pkgset_repos": {
"type": "object",
--
2.13.6

64
0008-Clone-module-defaults-into-work-directory.patch Normal file
View File

@ -0,0 +1,64 @@
From 9104444a7bc874c18de2a57851356e60a776341c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
Date: Wed, 11 Apr 2018 16:05:08 +0200
Subject: [PATCH 8/8] Clone module defaults into work/ directory
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
We can not rely on config_dir being writable, and should not modify
anything in there anyway.
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
---
pungi/paths.py | 8 ++++++++
pungi/phases/createrepo.py | 2 +-
pungi/phases/init.py | 3 +--
3 files changed, 10 insertions(+), 3 deletions(-)
diff --git a/pungi/paths.py b/pungi/paths.py
index 0e506795..37c6da13 100644
--- a/pungi/paths.py
+++ b/pungi/paths.py
@@ -340,6 +340,14 @@ class WorkPaths(object):
path = "%s.cfg" % path
return path
+ def module_defaults_dir(self, create_dir=True):
+ """
+ """
+ path = os.path.join(self.topdir(create_dir=create_dir), 'module_defaults')
+ if create_dir:
+ makedirs(path)
+ return path
+
class ComposePaths(object):
def __init__(self, compose):
diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py
index 49ff553a..f8e61387 100644
--- a/pungi/phases/createrepo.py
+++ b/pungi/phases/createrepo.py
@@ -219,7 +219,7 @@ def create_variant_repo(compose, arch, variant, pkg_type, modules_metadata=None)
modules.append(repo_mmd)
module_names = set([x.get_name() for x in modules])
- for mmddeffile in glob.glob(os.path.join(compose.config_dir, "module_defaults", "*.yaml")):
+ for mmddeffile in glob.glob(os.path.join(compose.paths.work.module_defaults_dir(), "*.yaml")):
for mmddef in Modulemd.objects_from_file(mmddeffile):
if isinstance(mmddef, Modulemd.Defaults) and mmddef.peek_module_name() in module_names:
modules.append(mmddef)
diff --git a/pungi/phases/init.py b/pungi/phases/init.py
index ef203568..9d9212e7 100644
--- a/pungi/phases/init.py
+++ b/pungi/phases/init.py
@@ -178,5 +178,4 @@ def write_module_defaults(compose):
with temp_dir(prefix="moduledefaults_") as tmp_dir:
get_dir_from_scm(scm_dict, tmp_dir, logger=compose._logger)
compose.log_debug("Writing module defaults")
- shutil.rmtree(os.path.join(compose.config_dir, "module_defaults"), ignore_errors=True)
- shutil.copytree(tmp_dir, os.path.join(compose.config_dir, "module_defaults"))
+ shutil.copytree(tmp_dir, compose.paths.work.module_defaults_dir())
--
2.13.6

11
pungi.spec
View File

@ -1,6 +1,6 @@
Name: pungi
Version: 4.1.23
Release: 2%{?dist}
Release: 3%{?dist}
Summary: Distribution compose tool
Group: Development/Tools
@ -11,6 +11,12 @@ Patch0: 0001-tests-Use-dummy-modulesdir-for-DNF.patch
Patch1: 0001-Revert-Move-ostree-phase-and-pipelines-for-running-p.patch
Patch2: 0002-Revert-Other-repo-for-OstreeInstaller.patch
Patch3: 0003-Revert-Ostree-can-use-pkgset-repos.patch
Patch4: 0004-Include-module-defaults-in-the-repodata.patch
Patch5: 0005-Handle-relative-paths-in-module_defaults_dir.patch
Patch6: 0006-Update-configuration-docs-with-module_defaults_dir.patch
Patch7: 0007-Update-the-configuration-JSON-schema-for-module_defa.patch
Patch8: 0008-Clone-module-defaults-into-work-directory.patch
BuildRequires: python3-nose
BuildRequires: python3-mock
BuildRequires: python2-devel
@ -176,6 +182,9 @@ nosetests-3 --exe
%{_bindir}/%{name}-wait-for-signed-ostree-handler
%changelog
* Thu Apr 12 2018 Lubomír Sedlář <lsedlar@redhat.com> - 4.1.23-3
- Add support for module defaults
* Wed Apr 11 2018 Lubomír Sedlář <lsedlar@redhat.com> - 4.1.23-2
- Revert reordering of ostree phases