Add support for module defaults

0001-Revert-Move-ostree-phase-and-pipelines-for-running-p.patch

@@ -1,7 +1,7 @@
 From 3f60e62ea86a3180b45290102d3519347cf8788a Mon Sep 17 00:00:00 2001
 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
 Date: Wed, 11 Apr 2018 09:18:59 +0200
-Subject: [PATCH 1/3] Revert "Move ostree phase and pipelines for running
+Subject: [PATCH 1/8] Revert "Move ostree phase and pipelines for running
  phases"
 
 This reverts commit 660c04368ba1abed310f121d01f0fa029eea5f11.

0002-Revert-Other-repo-for-OstreeInstaller.patch

@@ -1,7 +1,7 @@
 From fcc431622fc0305248bb966271c71c8d17fd5f69 Mon Sep 17 00:00:00 2001
 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
 Date: Wed, 11 Apr 2018 09:19:53 +0200
-Subject: [PATCH 2/3] Revert "Other repo for OstreeInstaller"
+Subject: [PATCH 2/8] Revert "Other repo for OstreeInstaller"
 
 This reverts commit 5c081cb545715c2a912ff50fa57554e89d905868.
 ---

0003-Revert-Ostree-can-use-pkgset-repos.patch

@@ -1,7 +1,7 @@
 From 8a7bf97434cbbf2397d3209498eacc326fc130f2 Mon Sep 17 00:00:00 2001
 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
 Date: Wed, 11 Apr 2018 09:20:51 +0200
-Subject: [PATCH 3/3] Revert "Ostree can use pkgset repos"
+Subject: [PATCH 3/8] Revert "Ostree can use pkgset repos"
 
 This reverts commit c7cc200246300c6a3946b2e3a9f5f7693896a7d6.
 ---

0004-Include-module-defaults-in-the-repodata.patch

@@ -0,0 +1,131 @@
+From 7d3baa8531e11059866412ddc6ae1aae4e855fc2 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Petr=20=C5=A0abata?= <contyk@redhat.com>
+Date: Wed, 4 Apr 2018 12:59:52 +0200
+Subject: [PATCH 4/8] Include module defaults in the repodata
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+If the compose configuration includes the module_defaults_dir (an
+scm_dict), clone the directory, read the module defaults contained
+therein and include relevant defaults in the combined modulemd file.
+
+Only defaults for modules present in the variant are included.
+
+This requires libmodulemd 1.2.0+.
+
+Merges: https://pagure.io/pungi/pull-request/891
+Signed-off-by: Petr Šabata <contyk@redhat.com>
+---
+ pungi/compose.py           |  4 ++++
+ pungi/phases/createrepo.py |  8 +++++++-
+ pungi/phases/init.py       | 17 ++++++++++++++++-
+ tests/test_initphase.py    |  3 +++
+ 4 files changed, 30 insertions(+), 2 deletions(-)
+
+diff --git a/pungi/compose.py b/pungi/compose.py
+index 4199d72a..07c67f4f 100644
+--- a/pungi/compose.py
++++ b/pungi/compose.py
+@@ -194,6 +194,10 @@ class Compose(kobo.log.LoggingBase):
+         return bool(self.conf.get("comps_file", False))
+ 
+     @property
++    def has_module_defaults(self):
++        return bool(self.conf.get("module_defaults_dir", False))
++
++    @property
+     def config_dir(self):
+         return os.path.dirname(self.conf._open_file or "")
+ 
+diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py
+index 1ce2b171..49ff553a 100644
+--- a/pungi/phases/createrepo.py
++++ b/pungi/phases/createrepo.py
+@@ -218,9 +218,15 @@ def create_variant_repo(compose, arch, variant, pkg_type, modules_metadata=None)
+                     raise AttributeError("module_metadata parameter was not passed and it is needed for module processing")
+             modules.append(repo_mmd)
+ 
++        module_names = set([x.get_name() for x in modules])
++        for mmddeffile in glob.glob(os.path.join(compose.config_dir, "module_defaults", "*.yaml")):
++            for mmddef in Modulemd.objects_from_file(mmddeffile):
++                if isinstance(mmddef, Modulemd.Defaults) and mmddef.peek_module_name() in module_names:
++                    modules.append(mmddef)
++
+         with temp_dir() as tmp_dir:
+             modules_path = os.path.join(tmp_dir, "modules.yaml")
+-            Modulemd.Module.dump_all(modules, modules_path)
++            Modulemd.dump(modules, modules_path)
+ 
+             cmd = repo.get_modifyrepo_cmd(os.path.join(repo_dir, "repodata"),
+                                           modules_path, mdtype="modules",
+diff --git a/pungi/phases/init.py b/pungi/phases/init.py
+index a01168a9..81eec0f4 100644
+--- a/pungi/phases/init.py
++++ b/pungi/phases/init.py
+@@ -23,7 +23,8 @@ from pungi.phases.base import PhaseBase
+ from pungi.phases.gather import write_prepopulate_file
+ from pungi.wrappers.createrepo import CreaterepoWrapper
+ from pungi.wrappers.comps import CompsWrapper
+-from pungi.wrappers.scm import get_file_from_scm
++from pungi.wrappers.scm import get_file_from_scm, get_dir_from_scm
++from pungi.util import temp_dir
+ 
+ 
+ class InitPhase(PhaseBase):
+@@ -63,6 +64,10 @@ class InitPhase(PhaseBase):
+ 
+         # download variants.xml / product.xml?
+ 
++        # download module defaults
++        if self.compose.has_module_defaults:
++            write_module_defaults(self.compose)
++
+         # write prepopulate file
+         write_prepopulate_file(self.compose)
+ 
+@@ -160,3 +165,13 @@ def create_comps_repo(compose, arch):
+                                       checksum=createrepo_checksum)
+         run(cmd, logfile=compose.paths.log.log_file(arch, "comps_repo"), show_cmd=True)
+         compose.log_info("[DONE ] %s" % msg)
++
++
++def write_module_defaults(compose):
++    scm_dict = compose.conf["module_defaults_dir"]
++
++    with temp_dir(prefix="moduledefaults_") as tmp_dir:
++        get_dir_from_scm(scm_dict, tmp_dir, logger=compose._logger)
++        compose.log_debug("Writing module defaults")
++        shutil.rmtree(os.path.join(compose.config_dir, "module_defaults"), ignore_errors=True)
++        shutil.copytree(tmp_dir, os.path.join(compose.config_dir, "module_defaults"))
+diff --git a/tests/test_initphase.py b/tests/test_initphase.py
+index ae6c4e52..8c3b0d9d 100644
+--- a/tests/test_initphase.py
++++ b/tests/test_initphase.py
+@@ -24,6 +24,7 @@ class TestInitPhase(PungiTestCase):
+     def test_run(self, write_prepopulate, write_variant, create_comps, write_arch, write_global):
+         compose = DummyCompose(self.topdir, {})
+         compose.has_comps = True
++        compose.has_module_defaults = False
+         compose.setup_optional()
+         phase = init.InitPhase(compose)
+         phase.run()
+@@ -52,6 +53,7 @@ class TestInitPhase(PungiTestCase):
+                                write_arch, write_global, copy_comps):
+         compose = DummyCompose(self.topdir, {})
+         compose.has_comps = True
++        compose.has_module_defaults = False
+         compose.variants['Everything'].groups = []
+         compose.variants['Everything'].modules = []
+         phase = init.InitPhase(compose)
+@@ -81,6 +83,7 @@ class TestInitPhase(PungiTestCase):
+                                write_arch, write_global, copy_comps):
+         compose = DummyCompose(self.topdir, {})
+         compose.has_comps = False
++        compose.has_module_defaults = False
+         phase = init.InitPhase(compose)
+         phase.run()
+ 
+-- 
+2.13.6
+

0005-Handle-relative-paths-in-module_defaults_dir.patch

@@ -0,0 +1,35 @@
+From 91a03c693222a41a23d1a3250984e0d8c9372b75 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Petr=20=C5=A0abata?= <contyk@redhat.com>
+Date: Wed, 4 Apr 2018 14:01:10 +0200
+Subject: [PATCH 5/8] Handle relative paths in module_defaults_dir
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+This now handles strings as well as sct_dicts with type=file that
+include relative paths.
+
+Signed-off-by: Petr Šabata <contyk@redhat.com>
+---
+ pungi/phases/init.py | 5 +++++
+ 1 file changed, 5 insertions(+)
+
+diff --git a/pungi/phases/init.py b/pungi/phases/init.py
+index 81eec0f4..ef203568 100644
+--- a/pungi/phases/init.py
++++ b/pungi/phases/init.py
+@@ -169,6 +169,11 @@ def create_comps_repo(compose, arch):
+ 
+ def write_module_defaults(compose):
+     scm_dict = compose.conf["module_defaults_dir"]
++    if isinstance(scm_dict, dict):
++        if scm_dict["scm"] == "file":
++            scm_dict["dir"] = os.path.join(compose.config_dir, scm_dict["dir"])
++    else:
++        scm_dict = os.path.join(compose.config_dir, scm_dict)
+ 
+     with temp_dir(prefix="moduledefaults_") as tmp_dir:
+         get_dir_from_scm(scm_dict, tmp_dir, logger=compose._logger)
+-- 
+2.13.6
+

0006-Update-configuration-docs-with-module_defaults_dir.patch

@@ -0,0 +1,39 @@
+From 587b4847843af878650d56e7c093cbef6b356801 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Petr=20=C5=A0abata?= <contyk@redhat.com>
+Date: Wed, 4 Apr 2018 14:02:15 +0200
+Subject: [PATCH 6/8] Update configuration docs with module_defaults_dir
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+Signed-off-by: Petr Šabata <contyk@redhat.com>
+---
+ doc/configuration.rst | 5 +++++
+ 1 file changed, 5 insertions(+)
+
+diff --git a/doc/configuration.rst b/doc/configuration.rst
+index e5f1e9e0..9e6cf1a9 100644
+--- a/doc/configuration.rst
++++ b/doc/configuration.rst
+@@ -21,6 +21,7 @@ Minimal Config Example
+     # GENERAL SETTINGS
+     comps_file = "comps-f23.xml"
+     variants_file = "variants-f23.xml"
++    module_defaults_dir = "module_defaults"
+ 
+     # KOJI
+     koji_profile = "koji"
+@@ -135,6 +136,10 @@ Options
+     (:ref:`scm_dict <scm_support>` or *str*) -- reference to variants XML file
+     that defines release variants and architectures
+ 
++**module_defaults_dir** [optional]
++    (:ref:`scm_dict <scm_support>` or *str*) -- reference the module defaults
++    directory containing modulemd-defaults YAML documents
++
+ **failable_deliverables** [optional]
+     (*list*) -- list which deliverables on which variant and architecture can
+     fail and not abort the whole compose. This only applies to ``buildinstall``
+-- 
+2.13.6
+

0007-Update-the-configuration-JSON-schema-for-module_defa.patch

@@ -0,0 +1,29 @@
+From dc6fccd522efc0ae0c2072812b6fc0559cc74475 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Petr=20=C5=A0abata?= <contyk@redhat.com>
+Date: Wed, 11 Apr 2018 10:33:42 +0200
+Subject: [PATCH 7/8] Update the configuration JSON schema for
+ module_defaults_dir
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+Signed-off-by: Petr Šabata <contyk@redhat.com>
+---
+ pungi/checks.py | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/pungi/checks.py b/pungi/checks.py
+index 189ca948..1c2059b2 100644
+--- a/pungi/checks.py
++++ b/pungi/checks.py
+@@ -678,6 +678,7 @@ def make_schema():
+                 "type": "boolean",
+                 "default": True
+             },
++            "module_defaults_dir": {"$ref": "#/definitions/str_or_scm_dict"},
+ 
+             "pkgset_repos": {
+                 "type": "object",
+-- 
+2.13.6
+

0008-Clone-module-defaults-into-work-directory.patch

@@ -0,0 +1,64 @@
+From 9104444a7bc874c18de2a57851356e60a776341c Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
+Date: Wed, 11 Apr 2018 16:05:08 +0200
+Subject: [PATCH 8/8] Clone module defaults into work/ directory
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+We can not rely on config_dir being writable, and should not modify
+anything in there anyway.
+
+Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
+---
+ pungi/paths.py             | 8 ++++++++
+ pungi/phases/createrepo.py | 2 +-
+ pungi/phases/init.py       | 3 +--
+ 3 files changed, 10 insertions(+), 3 deletions(-)
+
+diff --git a/pungi/paths.py b/pungi/paths.py
+index 0e506795..37c6da13 100644
+--- a/pungi/paths.py
++++ b/pungi/paths.py
+@@ -340,6 +340,14 @@ class WorkPaths(object):
+         path = "%s.cfg" % path
+         return path
+ 
++    def module_defaults_dir(self, create_dir=True):
++        """
++        """
++        path = os.path.join(self.topdir(create_dir=create_dir), 'module_defaults')
++        if create_dir:
++            makedirs(path)
++        return path
++
+ 
+ class ComposePaths(object):
+     def __init__(self, compose):
+diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py
+index 49ff553a..f8e61387 100644
+--- a/pungi/phases/createrepo.py
++++ b/pungi/phases/createrepo.py
+@@ -219,7 +219,7 @@ def create_variant_repo(compose, arch, variant, pkg_type, modules_metadata=None)
+             modules.append(repo_mmd)
+ 
+         module_names = set([x.get_name() for x in modules])
+-        for mmddeffile in glob.glob(os.path.join(compose.config_dir, "module_defaults", "*.yaml")):
++        for mmddeffile in glob.glob(os.path.join(compose.paths.work.module_defaults_dir(), "*.yaml")):
+             for mmddef in Modulemd.objects_from_file(mmddeffile):
+                 if isinstance(mmddef, Modulemd.Defaults) and mmddef.peek_module_name() in module_names:
+                     modules.append(mmddef)
+diff --git a/pungi/phases/init.py b/pungi/phases/init.py
+index ef203568..9d9212e7 100644
+--- a/pungi/phases/init.py
++++ b/pungi/phases/init.py
+@@ -178,5 +178,4 @@ def write_module_defaults(compose):
+     with temp_dir(prefix="moduledefaults_") as tmp_dir:
+         get_dir_from_scm(scm_dict, tmp_dir, logger=compose._logger)
+         compose.log_debug("Writing module defaults")
+-        shutil.rmtree(os.path.join(compose.config_dir, "module_defaults"), ignore_errors=True)
+-        shutil.copytree(tmp_dir, os.path.join(compose.config_dir, "module_defaults"))
++        shutil.copytree(tmp_dir, compose.paths.work.module_defaults_dir())
+-- 
+2.13.6
+

pungi.spec

@@ -1,6 +1,6 @@
 Name:           pungi
 Version:        4.1.23
-Release:        2%{?dist}
+Release:        3%{?dist}
 Summary:        Distribution compose tool
 
 Group:          Development/Tools
@@ -11,6 +11,12 @@ Patch0:         0001-tests-Use-dummy-modulesdir-for-DNF.patch
 Patch1:         0001-Revert-Move-ostree-phase-and-pipelines-for-running-p.patch
 Patch2:         0002-Revert-Other-repo-for-OstreeInstaller.patch
 Patch3:         0003-Revert-Ostree-can-use-pkgset-repos.patch
+Patch4:         0004-Include-module-defaults-in-the-repodata.patch
+Patch5:         0005-Handle-relative-paths-in-module_defaults_dir.patch
+Patch6:         0006-Update-configuration-docs-with-module_defaults_dir.patch
+Patch7:         0007-Update-the-configuration-JSON-schema-for-module_defa.patch
+Patch8:         0008-Clone-module-defaults-into-work-directory.patch
+
 BuildRequires:  python3-nose
 BuildRequires:  python3-mock
 BuildRequires:  python2-devel
@@ -176,6 +182,9 @@ nosetests-3 --exe
 %{_bindir}/%{name}-wait-for-signed-ostree-handler
 
 %changelog
+* Thu Apr 12 2018 Lubomír Sedlář <lsedlar@redhat.com> - 4.1.23-3
+- Add support for module defaults
+
 * Wed Apr 11 2018 Lubomír Sedlář <lsedlar@redhat.com> - 4.1.23-2
 - Revert reordering of ostree phases