diff --git a/pungi/__init__.py b/pungi/__init__.py index 22aaf88a..1b5dc482 100644 --- a/pungi/__init__.py +++ b/pungi/__init__.py @@ -4,14 +4,6 @@ import os import re -try: - import gi - gi.require_version('Modulemd', '2.0') # noqa - from gi.repository import Modulemd -except (ImportError, ValueError): - Modulemd = None - - def get_full_version(): """ Find full version of Pungi: if running from git, this will return cleaned diff --git a/pungi/module_util.py b/pungi/module_util.py new file mode 100644 index 00000000..ab29a67c --- /dev/null +++ b/pungi/module_util.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . + +import glob +import os + +try: + import gi + + gi.require_version("Modulemd", "2.0") # noqa + from gi.repository import Modulemd +except (ImportError, ValueError): + Modulemd = None + + +def iter_module_defaults(path): + """Given a path to a directory with yaml files, yield each module default + in there as a pair (module_name, ModuleDefaults instance). + """ + # It is really tempting to merge all the module indexes into a single one + # and work with it. However that does not allow for detecting conflicting + # defaults. That should not happen in practice, but better safe than sorry. + # Once libmodulemd can report the error, this code can be simplifed by a + # lot. It was implemented in + # https://github.com/fedora-modularity/libmodulemd/commit/3087e4a5c38a331041fec9b6b8f1a372f9ffe64d + # and released in 2.6.0, but 2.8.0 added the need to merge overrides and + # that breaks this use case again. + for file in glob.glob(os.path.join(path, "*.yaml")): + index = Modulemd.ModuleIndex() + index.update_from_file(file, strict=False) + for module_name in index.get_module_names(): + yield module_name, index.get_module(module_name).get_defaults() + + +def collect_module_defaults( + defaults_dir, modules_to_load=None, mod_index=None, overrides_dir=None +): + """Load module defaults into index. + + If `modules_to_load` is passed in, it should be a set of module names. Only + defaults for these modules will be loaded. + + If `mod_index` is passed in, it will be updated and returned. If it was + not, a new ModuleIndex will be created and returned + """ + mod_index = mod_index or Modulemd.ModuleIndex() + + temp_index = Modulemd.ModuleIndex.new() + temp_index.update_from_defaults_directory( + defaults_dir, overrides_path=overrides_dir, strict=False + ) + + for module_name in temp_index.get_module_names(): + defaults = temp_index.get_module(module_name).get_defaults() + + if not modules_to_load or module_name in modules_to_load: + mod_index.add_defaults(defaults) + + return mod_index diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py index b2c67ad7..e0a1da9c 100644 --- a/pungi/phases/createrepo.py +++ b/pungi/phases/createrepo.py @@ -33,13 +33,8 @@ from kobo.shortcuts import run, relative_path from ..wrappers.scm import get_dir_from_scm from ..wrappers.createrepo import CreaterepoWrapper from .base import PhaseBase -from ..util import ( - find_old_compose, - get_arch_variant_data, - collect_module_defaults, - temp_dir, -) -from pungi import Modulemd +from ..util import find_old_compose, get_arch_variant_data, temp_dir +from ..module_util import Modulemd, collect_module_defaults import productmd.rpms import productmd.modules diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py index 736c10ec..2deda172 100644 --- a/pungi/phases/gather/__init__.py +++ b/pungi/phases/gather/__init__.py @@ -27,12 +27,12 @@ from .link import link_files from ...wrappers.createrepo import CreaterepoWrapper import pungi.wrappers.kojiwrapper -from pungi import Modulemd from pungi.compose import get_ordered_variant_uids from pungi.arch import get_compatible_arches, split_name_arch from pungi.phases.base import PhaseBase from pungi.util import (get_arch_data, get_arch_variant_data, get_variant_data, - makedirs, collect_module_defaults) + makedirs) +from pungi.module_util import Modulemd, collect_module_defaults from pungi.phases.createrepo import add_modular_metadata diff --git a/pungi/phases/gather/methods/method_hybrid.py b/pungi/phases/gather/methods/method_hybrid.py index ade3c4e8..448fd267 100644 --- a/pungi/phases/gather/methods/method_hybrid.py +++ b/pungi/phases/gather/methods/method_hybrid.py @@ -23,13 +23,11 @@ import kobo.rpmlib from kobo.shortcuts import run import pungi.phases.gather.method -from pungi import Modulemd, multilib_dnf +from pungi import multilib_dnf +from pungi.module_util import Modulemd from pungi.arch import get_valid_arches, tree_arch_to_yum_arch from pungi.phases.gather import _mk_pkg_map -from pungi.util import ( - get_arch_variant_data, - pkg_is_debug, -) +from pungi.util import get_arch_variant_data, pkg_is_debug from pungi.wrappers import fus from pungi.wrappers.comps import CompsWrapper diff --git a/pungi/phases/init.py b/pungi/phases/init.py index 3d5472fa..12969544 100644 --- a/pungi/phases/init.py +++ b/pungi/phases/init.py @@ -23,7 +23,8 @@ from kobo.threads import run_in_threads from pungi.phases.base import PhaseBase from pungi.phases.gather import write_prepopulate_file -from pungi.util import temp_dir, iter_module_defaults +from pungi.util import temp_dir +from pungi.module_util import iter_module_defaults from pungi.wrappers.comps import CompsWrapper from pungi.wrappers.createrepo import CreaterepoWrapper from pungi.wrappers.scm import get_dir_from_scm, get_file_from_scm diff --git a/pungi/phases/pkgset/common.py b/pungi/phases/pkgset/common.py index d7e06cfc..6e53cfc9 100644 --- a/pungi/phases/pkgset/common.py +++ b/pungi/phases/pkgset/common.py @@ -20,10 +20,10 @@ import threading from kobo.shortcuts import run, relative_path from kobo.threads import run_in_threads -from pungi import Modulemd from pungi.arch import get_valid_arches from pungi.wrappers.createrepo import CreaterepoWrapper -from pungi.util import is_arch_multilib, find_old_compose, collect_module_defaults +from pungi.util import is_arch_multilib, find_old_compose +from pungi.module_util import Modulemd, collect_module_defaults from pungi.phases.createrepo import add_modular_metadata diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index 3a0dfb52..187e6f1b 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -28,7 +28,7 @@ from pungi.wrappers.comps import CompsWrapper import pungi.phases.pkgset.pkgsets from pungi.arch import getBaseArch from pungi.util import retry, find_old_compose, get_arch_variant_data -from pungi import Modulemd +from pungi.module_util import Modulemd from pungi.phases.pkgset.common import MaterializedPackageSet, get_all_arches from pungi.phases.gather import get_packages_to_gather diff --git a/pungi/util.py b/pungi/util.py index bd3e022c..8bafec48 100644 --- a/pungi/util.py +++ b/pungi/util.py @@ -15,7 +15,6 @@ import argparse import fnmatch -import glob import json import subprocess import os @@ -36,8 +35,6 @@ import kobo.conf from kobo.shortcuts import run, force_list from productmd.common import get_major_version -from pungi import Modulemd - # Patterns that match all names of debuginfo packages DEBUG_PATTERNS = ["*-debuginfo", "*-debuginfo-*", "*-debugsource"] @@ -926,52 +923,6 @@ def parse_koji_event(event): ) -def iter_module_defaults(path): - """Given a path to a directory with yaml files, yield each module default - in there as a pair (module_name, ModuleDefaults instance). - """ - # It is really tempting to merge all the module indexes into a single one - # and work with it. However that does not allow for detecting conflicting - # defaults. That should not happen in practice, but better safe than sorry. - # Once libmodulemd can report the error, this code can be simplifed by a - # lot. It was implemented in - # https://github.com/fedora-modularity/libmodulemd/commit/3087e4a5c38a331041fec9b6b8f1a372f9ffe64d - # and released in 2.6.0, but 2.8.0 added the need to merge overrides and - # that breaks this use case again. - for file in glob.glob(os.path.join(path, "*.yaml")): - index = Modulemd.ModuleIndex() - index.update_from_file(file, strict=False) - for module_name in index.get_module_names(): - yield module_name, index.get_module(module_name).get_defaults() - - -def collect_module_defaults( - defaults_dir, modules_to_load=None, mod_index=None, overrides_dir=None -): - """Load module defaults into index. - - If `modules_to_load` is passed in, it should be a set of module names. Only - defaults for these modules will be loaded. - - If `mod_index` is passed in, it will be updated and returned. If it was - not, a new ModuleIndex will be created and returned - """ - mod_index = mod_index or Modulemd.ModuleIndex() - - temp_index = Modulemd.ModuleIndex.new() - temp_index.update_from_defaults_directory( - defaults_dir, overrides_path=overrides_dir, strict=False - ) - - for module_name in temp_index.get_module_names(): - defaults = temp_index.get_module(module_name).get_defaults() - - if not modules_to_load or module_name in modules_to_load: - mod_index.add_defaults(defaults) - - return mod_index - - def load_config(file_path, defaults={}): """Open and load configuration file form .conf or .json file.""" conf = kobo.conf.PyConfigParser() diff --git a/tests/helpers.py b/tests/helpers.py index ea1b4936..9650d401 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -18,7 +18,8 @@ except ImportError: import unittest from pungi.util import get_arch_variant_data -from pungi import paths, checks, Modulemd +from pungi import paths, checks +from pungi.module_util import Modulemd class BaseTestCase(unittest.TestCase): diff --git a/tests/test_createrepophase.py b/tests/test_createrepophase.py index cc3e9488..c7c43c5d 100644 --- a/tests/test_createrepophase.py +++ b/tests/test_createrepophase.py @@ -19,7 +19,7 @@ from pungi.phases.createrepo import (CreaterepoPhase, get_productids_from_scm, ModulesMetadata) from tests.helpers import DummyCompose, PungiTestCase, copy_fixture, touch -from pungi import Modulemd +from pungi.module_util import Modulemd class TestCreaterepoPhase(PungiTestCase): diff --git a/tests/test_gather_source_module.py b/tests/test_gather_source_module.py index 400cc9c5..9446bc80 100644 --- a/tests/test_gather_source_module.py +++ b/tests/test_gather_source_module.py @@ -13,7 +13,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) from pungi.phases.gather.sources.source_module import GatherSourceModule from tests import helpers -from pungi import Modulemd +from pungi.module_util import Modulemd @unittest.skipUnless(Modulemd is not None, "Skipped test, no module support.") diff --git a/tests/test_initphase.py b/tests/test_initphase.py index 4783e230..964913d6 100644 --- a/tests/test_initphase.py +++ b/tests/test_initphase.py @@ -13,7 +13,7 @@ import sys sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) -from pungi import Modulemd +from pungi.module_util import Modulemd from pungi.phases import init from tests.helpers import DummyCompose, PungiTestCase, touch, mk_boom, fake_run_in_threads diff --git a/tests/test_pkgset_common.py b/tests/test_pkgset_common.py index d1e147fe..03321a9c 100755 --- a/tests/test_pkgset_common.py +++ b/tests/test_pkgset_common.py @@ -8,7 +8,7 @@ import mock sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) -from pungi import Modulemd +from pungi.module_util import Modulemd from pungi.phases.pkgset import common from tests import helpers diff --git a/tests/test_pkgset_source_koji.py b/tests/test_pkgset_source_koji.py index bd939683..a9d7fb3d 100644 --- a/tests/test_pkgset_source_koji.py +++ b/tests/test_pkgset_source_koji.py @@ -14,7 +14,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) from pungi.phases.pkgset.sources import source_koji from tests import helpers -from pungi import Modulemd +from pungi.module_util import Modulemd EVENT_INFO = {'id': 15681980, 'ts': 1460956382.81936} TAG_INFO = { @@ -613,7 +613,7 @@ class MockModule(object): return self.path == other.path -@mock.patch("pungi.Modulemd.ModuleStream.read_file", new=MockModule) +@mock.patch("pungi.module_util.Modulemd.ModuleStream.read_file", new=MockModule) @unittest.skipIf(Modulemd is None, "Skipping tests, no module support") class TestAddModuleToVariant(helpers.PungiTestCase):