From 6c327d69719141e2bcf3ebf942e77b3775f68f97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Thu, 12 Apr 2018 11:04:22 +0200 Subject: [PATCH] Add support for module defaults --- ...ee-phase-and-pipelines-for-running-p.patch | 2 +- ...evert-Other-repo-for-OstreeInstaller.patch | 2 +- 0003-Revert-Ostree-can-use-pkgset-repos.patch | 2 +- ...lude-module-defaults-in-the-repodata.patch | 131 ++++++++++++++++++ ...elative-paths-in-module_defaults_dir.patch | 35 +++++ ...ration-docs-with-module_defaults_dir.patch | 39 ++++++ ...guration-JSON-schema-for-module_defa.patch | 29 ++++ ...-module-defaults-into-work-directory.patch | 64 +++++++++ pungi.spec | 11 +- 9 files changed, 311 insertions(+), 4 deletions(-) create mode 100644 0004-Include-module-defaults-in-the-repodata.patch create mode 100644 0005-Handle-relative-paths-in-module_defaults_dir.patch create mode 100644 0006-Update-configuration-docs-with-module_defaults_dir.patch create mode 100644 0007-Update-the-configuration-JSON-schema-for-module_defa.patch create mode 100644 0008-Clone-module-defaults-into-work-directory.patch diff --git a/0001-Revert-Move-ostree-phase-and-pipelines-for-running-p.patch b/0001-Revert-Move-ostree-phase-and-pipelines-for-running-p.patch index 9714a790..e59620a8 100644 --- a/0001-Revert-Move-ostree-phase-and-pipelines-for-running-p.patch +++ b/0001-Revert-Move-ostree-phase-and-pipelines-for-running-p.patch @@ -1,7 +1,7 @@ From 3f60e62ea86a3180b45290102d3519347cf8788a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Wed, 11 Apr 2018 09:18:59 +0200 -Subject: [PATCH 1/3] Revert "Move ostree phase and pipelines for running +Subject: [PATCH 1/8] Revert "Move ostree phase and pipelines for running phases" This reverts commit 660c04368ba1abed310f121d01f0fa029eea5f11. diff --git a/0002-Revert-Other-repo-for-OstreeInstaller.patch b/0002-Revert-Other-repo-for-OstreeInstaller.patch index 30c27f06..6a0259ef 100644 --- a/0002-Revert-Other-repo-for-OstreeInstaller.patch +++ b/0002-Revert-Other-repo-for-OstreeInstaller.patch @@ -1,7 +1,7 @@ From fcc431622fc0305248bb966271c71c8d17fd5f69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Wed, 11 Apr 2018 09:19:53 +0200 -Subject: [PATCH 2/3] Revert "Other repo for OstreeInstaller" +Subject: [PATCH 2/8] Revert "Other repo for OstreeInstaller" This reverts commit 5c081cb545715c2a912ff50fa57554e89d905868. --- diff --git a/0003-Revert-Ostree-can-use-pkgset-repos.patch b/0003-Revert-Ostree-can-use-pkgset-repos.patch index 3e8e08e5..f8262e82 100644 --- a/0003-Revert-Ostree-can-use-pkgset-repos.patch +++ b/0003-Revert-Ostree-can-use-pkgset-repos.patch @@ -1,7 +1,7 @@ From 8a7bf97434cbbf2397d3209498eacc326fc130f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Wed, 11 Apr 2018 09:20:51 +0200 -Subject: [PATCH 3/3] Revert "Ostree can use pkgset repos" +Subject: [PATCH 3/8] Revert "Ostree can use pkgset repos" This reverts commit c7cc200246300c6a3946b2e3a9f5f7693896a7d6. --- diff --git a/0004-Include-module-defaults-in-the-repodata.patch b/0004-Include-module-defaults-in-the-repodata.patch new file mode 100644 index 00000000..eb09682a --- /dev/null +++ b/0004-Include-module-defaults-in-the-repodata.patch @@ -0,0 +1,131 @@ +From 7d3baa8531e11059866412ddc6ae1aae4e855fc2 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Petr=20=C5=A0abata?= +Date: Wed, 4 Apr 2018 12:59:52 +0200 +Subject: [PATCH 4/8] Include module defaults in the repodata +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +If the compose configuration includes the module_defaults_dir (an +scm_dict), clone the directory, read the module defaults contained +therein and include relevant defaults in the combined modulemd file. + +Only defaults for modules present in the variant are included. + +This requires libmodulemd 1.2.0+. + +Merges: https://pagure.io/pungi/pull-request/891 +Signed-off-by: Petr Šabata +--- + pungi/compose.py | 4 ++++ + pungi/phases/createrepo.py | 8 +++++++- + pungi/phases/init.py | 17 ++++++++++++++++- + tests/test_initphase.py | 3 +++ + 4 files changed, 30 insertions(+), 2 deletions(-) + +diff --git a/pungi/compose.py b/pungi/compose.py +index 4199d72a..07c67f4f 100644 +--- a/pungi/compose.py ++++ b/pungi/compose.py +@@ -194,6 +194,10 @@ class Compose(kobo.log.LoggingBase): + return bool(self.conf.get("comps_file", False)) + + @property ++ def has_module_defaults(self): ++ return bool(self.conf.get("module_defaults_dir", False)) ++ ++ @property + def config_dir(self): + return os.path.dirname(self.conf._open_file or "") + +diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py +index 1ce2b171..49ff553a 100644 +--- a/pungi/phases/createrepo.py ++++ b/pungi/phases/createrepo.py +@@ -218,9 +218,15 @@ def create_variant_repo(compose, arch, variant, pkg_type, modules_metadata=None) + raise AttributeError("module_metadata parameter was not passed and it is needed for module processing") + modules.append(repo_mmd) + ++ module_names = set([x.get_name() for x in modules]) ++ for mmddeffile in glob.glob(os.path.join(compose.config_dir, "module_defaults", "*.yaml")): ++ for mmddef in Modulemd.objects_from_file(mmddeffile): ++ if isinstance(mmddef, Modulemd.Defaults) and mmddef.peek_module_name() in module_names: ++ modules.append(mmddef) ++ + with temp_dir() as tmp_dir: + modules_path = os.path.join(tmp_dir, "modules.yaml") +- Modulemd.Module.dump_all(modules, modules_path) ++ Modulemd.dump(modules, modules_path) + + cmd = repo.get_modifyrepo_cmd(os.path.join(repo_dir, "repodata"), + modules_path, mdtype="modules", +diff --git a/pungi/phases/init.py b/pungi/phases/init.py +index a01168a9..81eec0f4 100644 +--- a/pungi/phases/init.py ++++ b/pungi/phases/init.py +@@ -23,7 +23,8 @@ from pungi.phases.base import PhaseBase + from pungi.phases.gather import write_prepopulate_file + from pungi.wrappers.createrepo import CreaterepoWrapper + from pungi.wrappers.comps import CompsWrapper +-from pungi.wrappers.scm import get_file_from_scm ++from pungi.wrappers.scm import get_file_from_scm, get_dir_from_scm ++from pungi.util import temp_dir + + + class InitPhase(PhaseBase): +@@ -63,6 +64,10 @@ class InitPhase(PhaseBase): + + # download variants.xml / product.xml? + ++ # download module defaults ++ if self.compose.has_module_defaults: ++ write_module_defaults(self.compose) ++ + # write prepopulate file + write_prepopulate_file(self.compose) + +@@ -160,3 +165,13 @@ def create_comps_repo(compose, arch): + checksum=createrepo_checksum) + run(cmd, logfile=compose.paths.log.log_file(arch, "comps_repo"), show_cmd=True) + compose.log_info("[DONE ] %s" % msg) ++ ++ ++def write_module_defaults(compose): ++ scm_dict = compose.conf["module_defaults_dir"] ++ ++ with temp_dir(prefix="moduledefaults_") as tmp_dir: ++ get_dir_from_scm(scm_dict, tmp_dir, logger=compose._logger) ++ compose.log_debug("Writing module defaults") ++ shutil.rmtree(os.path.join(compose.config_dir, "module_defaults"), ignore_errors=True) ++ shutil.copytree(tmp_dir, os.path.join(compose.config_dir, "module_defaults")) +diff --git a/tests/test_initphase.py b/tests/test_initphase.py +index ae6c4e52..8c3b0d9d 100644 +--- a/tests/test_initphase.py ++++ b/tests/test_initphase.py +@@ -24,6 +24,7 @@ class TestInitPhase(PungiTestCase): + def test_run(self, write_prepopulate, write_variant, create_comps, write_arch, write_global): + compose = DummyCompose(self.topdir, {}) + compose.has_comps = True ++ compose.has_module_defaults = False + compose.setup_optional() + phase = init.InitPhase(compose) + phase.run() +@@ -52,6 +53,7 @@ class TestInitPhase(PungiTestCase): + write_arch, write_global, copy_comps): + compose = DummyCompose(self.topdir, {}) + compose.has_comps = True ++ compose.has_module_defaults = False + compose.variants['Everything'].groups = [] + compose.variants['Everything'].modules = [] + phase = init.InitPhase(compose) +@@ -81,6 +83,7 @@ class TestInitPhase(PungiTestCase): + write_arch, write_global, copy_comps): + compose = DummyCompose(self.topdir, {}) + compose.has_comps = False ++ compose.has_module_defaults = False + phase = init.InitPhase(compose) + phase.run() + +-- +2.13.6 + diff --git a/0005-Handle-relative-paths-in-module_defaults_dir.patch b/0005-Handle-relative-paths-in-module_defaults_dir.patch new file mode 100644 index 00000000..34cd05e1 --- /dev/null +++ b/0005-Handle-relative-paths-in-module_defaults_dir.patch @@ -0,0 +1,35 @@ +From 91a03c693222a41a23d1a3250984e0d8c9372b75 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Petr=20=C5=A0abata?= +Date: Wed, 4 Apr 2018 14:01:10 +0200 +Subject: [PATCH 5/8] Handle relative paths in module_defaults_dir +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +This now handles strings as well as sct_dicts with type=file that +include relative paths. + +Signed-off-by: Petr Šabata +--- + pungi/phases/init.py | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/pungi/phases/init.py b/pungi/phases/init.py +index 81eec0f4..ef203568 100644 +--- a/pungi/phases/init.py ++++ b/pungi/phases/init.py +@@ -169,6 +169,11 @@ def create_comps_repo(compose, arch): + + def write_module_defaults(compose): + scm_dict = compose.conf["module_defaults_dir"] ++ if isinstance(scm_dict, dict): ++ if scm_dict["scm"] == "file": ++ scm_dict["dir"] = os.path.join(compose.config_dir, scm_dict["dir"]) ++ else: ++ scm_dict = os.path.join(compose.config_dir, scm_dict) + + with temp_dir(prefix="moduledefaults_") as tmp_dir: + get_dir_from_scm(scm_dict, tmp_dir, logger=compose._logger) +-- +2.13.6 + diff --git a/0006-Update-configuration-docs-with-module_defaults_dir.patch b/0006-Update-configuration-docs-with-module_defaults_dir.patch new file mode 100644 index 00000000..d99ee2a3 --- /dev/null +++ b/0006-Update-configuration-docs-with-module_defaults_dir.patch @@ -0,0 +1,39 @@ +From 587b4847843af878650d56e7c093cbef6b356801 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Petr=20=C5=A0abata?= +Date: Wed, 4 Apr 2018 14:02:15 +0200 +Subject: [PATCH 6/8] Update configuration docs with module_defaults_dir +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Signed-off-by: Petr Šabata +--- + doc/configuration.rst | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/doc/configuration.rst b/doc/configuration.rst +index e5f1e9e0..9e6cf1a9 100644 +--- a/doc/configuration.rst ++++ b/doc/configuration.rst +@@ -21,6 +21,7 @@ Minimal Config Example + # GENERAL SETTINGS + comps_file = "comps-f23.xml" + variants_file = "variants-f23.xml" ++ module_defaults_dir = "module_defaults" + + # KOJI + koji_profile = "koji" +@@ -135,6 +136,10 @@ Options + (:ref:`scm_dict ` or *str*) -- reference to variants XML file + that defines release variants and architectures + ++**module_defaults_dir** [optional] ++ (:ref:`scm_dict ` or *str*) -- reference the module defaults ++ directory containing modulemd-defaults YAML documents ++ + **failable_deliverables** [optional] + (*list*) -- list which deliverables on which variant and architecture can + fail and not abort the whole compose. This only applies to ``buildinstall`` +-- +2.13.6 + diff --git a/0007-Update-the-configuration-JSON-schema-for-module_defa.patch b/0007-Update-the-configuration-JSON-schema-for-module_defa.patch new file mode 100644 index 00000000..b74198fb --- /dev/null +++ b/0007-Update-the-configuration-JSON-schema-for-module_defa.patch @@ -0,0 +1,29 @@ +From dc6fccd522efc0ae0c2072812b6fc0559cc74475 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Petr=20=C5=A0abata?= +Date: Wed, 11 Apr 2018 10:33:42 +0200 +Subject: [PATCH 7/8] Update the configuration JSON schema for + module_defaults_dir +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Signed-off-by: Petr Šabata +--- + pungi/checks.py | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/pungi/checks.py b/pungi/checks.py +index 189ca948..1c2059b2 100644 +--- a/pungi/checks.py ++++ b/pungi/checks.py +@@ -678,6 +678,7 @@ def make_schema(): + "type": "boolean", + "default": True + }, ++ "module_defaults_dir": {"$ref": "#/definitions/str_or_scm_dict"}, + + "pkgset_repos": { + "type": "object", +-- +2.13.6 + diff --git a/0008-Clone-module-defaults-into-work-directory.patch b/0008-Clone-module-defaults-into-work-directory.patch new file mode 100644 index 00000000..78f77bf9 --- /dev/null +++ b/0008-Clone-module-defaults-into-work-directory.patch @@ -0,0 +1,64 @@ +From 9104444a7bc874c18de2a57851356e60a776341c Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= +Date: Wed, 11 Apr 2018 16:05:08 +0200 +Subject: [PATCH 8/8] Clone module defaults into work/ directory +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +We can not rely on config_dir being writable, and should not modify +anything in there anyway. + +Signed-off-by: Lubomír Sedlář +--- + pungi/paths.py | 8 ++++++++ + pungi/phases/createrepo.py | 2 +- + pungi/phases/init.py | 3 +-- + 3 files changed, 10 insertions(+), 3 deletions(-) + +diff --git a/pungi/paths.py b/pungi/paths.py +index 0e506795..37c6da13 100644 +--- a/pungi/paths.py ++++ b/pungi/paths.py +@@ -340,6 +340,14 @@ class WorkPaths(object): + path = "%s.cfg" % path + return path + ++ def module_defaults_dir(self, create_dir=True): ++ """ ++ """ ++ path = os.path.join(self.topdir(create_dir=create_dir), 'module_defaults') ++ if create_dir: ++ makedirs(path) ++ return path ++ + + class ComposePaths(object): + def __init__(self, compose): +diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py +index 49ff553a..f8e61387 100644 +--- a/pungi/phases/createrepo.py ++++ b/pungi/phases/createrepo.py +@@ -219,7 +219,7 @@ def create_variant_repo(compose, arch, variant, pkg_type, modules_metadata=None) + modules.append(repo_mmd) + + module_names = set([x.get_name() for x in modules]) +- for mmddeffile in glob.glob(os.path.join(compose.config_dir, "module_defaults", "*.yaml")): ++ for mmddeffile in glob.glob(os.path.join(compose.paths.work.module_defaults_dir(), "*.yaml")): + for mmddef in Modulemd.objects_from_file(mmddeffile): + if isinstance(mmddef, Modulemd.Defaults) and mmddef.peek_module_name() in module_names: + modules.append(mmddef) +diff --git a/pungi/phases/init.py b/pungi/phases/init.py +index ef203568..9d9212e7 100644 +--- a/pungi/phases/init.py ++++ b/pungi/phases/init.py +@@ -178,5 +178,4 @@ def write_module_defaults(compose): + with temp_dir(prefix="moduledefaults_") as tmp_dir: + get_dir_from_scm(scm_dict, tmp_dir, logger=compose._logger) + compose.log_debug("Writing module defaults") +- shutil.rmtree(os.path.join(compose.config_dir, "module_defaults"), ignore_errors=True) +- shutil.copytree(tmp_dir, os.path.join(compose.config_dir, "module_defaults")) ++ shutil.copytree(tmp_dir, compose.paths.work.module_defaults_dir()) +-- +2.13.6 + diff --git a/pungi.spec b/pungi.spec index 4d24564b..d6f8bc23 100644 --- a/pungi.spec +++ b/pungi.spec @@ -1,6 +1,6 @@ Name: pungi Version: 4.1.23 -Release: 2%{?dist} +Release: 3%{?dist} Summary: Distribution compose tool Group: Development/Tools @@ -11,6 +11,12 @@ Patch0: 0001-tests-Use-dummy-modulesdir-for-DNF.patch Patch1: 0001-Revert-Move-ostree-phase-and-pipelines-for-running-p.patch Patch2: 0002-Revert-Other-repo-for-OstreeInstaller.patch Patch3: 0003-Revert-Ostree-can-use-pkgset-repos.patch +Patch4: 0004-Include-module-defaults-in-the-repodata.patch +Patch5: 0005-Handle-relative-paths-in-module_defaults_dir.patch +Patch6: 0006-Update-configuration-docs-with-module_defaults_dir.patch +Patch7: 0007-Update-the-configuration-JSON-schema-for-module_defa.patch +Patch8: 0008-Clone-module-defaults-into-work-directory.patch + BuildRequires: python3-nose BuildRequires: python3-mock BuildRequires: python2-devel @@ -176,6 +182,9 @@ nosetests-3 --exe %{_bindir}/%{name}-wait-for-signed-ostree-handler %changelog +* Thu Apr 12 2018 Lubomír Sedlář - 4.1.23-3 +- Add support for module defaults + * Wed Apr 11 2018 Lubomír Sedlář - 4.1.23-2 - Revert reordering of ostree phases