From 8ab7d9f7ba00a0e382502830515a0b43250b521e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Wed, 2 Oct 2019 09:40:18 +0200 Subject: [PATCH] Move import of modulemd to a separate module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This should make it possible to only import the library only when it's really needed. DNF does not work with libmodulemd v2. If we import libmodulemd2 and then dnf, the program will just hang forever. We only need DNF in pungi-gather, where libmodulemd is not needed, and also where we do need libmodulemd, we don't have DNF. Signed-off-by: Lubomír Sedlář --- pungi/__init__.py | 8 --- pungi/module_util.py | 71 ++++++++++++++++++++ pungi/phases/createrepo.py | 9 +-- pungi/phases/gather/__init__.py | 4 +- pungi/phases/gather/methods/method_hybrid.py | 8 +-- pungi/phases/init.py | 3 +- pungi/phases/pkgset/common.py | 4 +- pungi/phases/pkgset/sources/source_koji.py | 2 +- pungi/util.py | 49 -------------- tests/helpers.py | 3 +- tests/test_createrepophase.py | 2 +- tests/test_gather_source_module.py | 2 +- tests/test_initphase.py | 2 +- tests/test_pkgset_common.py | 2 +- tests/test_pkgset_source_koji.py | 4 +- 15 files changed, 91 insertions(+), 82 deletions(-) create mode 100644 pungi/module_util.py diff --git a/pungi/__init__.py b/pungi/__init__.py index 22aaf88a..1b5dc482 100644 --- a/pungi/__init__.py +++ b/pungi/__init__.py @@ -4,14 +4,6 @@ import os import re -try: - import gi - gi.require_version('Modulemd', '2.0') # noqa - from gi.repository import Modulemd -except (ImportError, ValueError): - Modulemd = None - - def get_full_version(): """ Find full version of Pungi: if running from git, this will return cleaned diff --git a/pungi/module_util.py b/pungi/module_util.py new file mode 100644 index 00000000..ab29a67c --- /dev/null +++ b/pungi/module_util.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . + +import glob +import os + +try: + import gi + + gi.require_version("Modulemd", "2.0") # noqa + from gi.repository import Modulemd +except (ImportError, ValueError): + Modulemd = None + + +def iter_module_defaults(path): + """Given a path to a directory with yaml files, yield each module default + in there as a pair (module_name, ModuleDefaults instance). + """ + # It is really tempting to merge all the module indexes into a single one + # and work with it. However that does not allow for detecting conflicting + # defaults. That should not happen in practice, but better safe than sorry. + # Once libmodulemd can report the error, this code can be simplifed by a + # lot. It was implemented in + # https://github.com/fedora-modularity/libmodulemd/commit/3087e4a5c38a331041fec9b6b8f1a372f9ffe64d + # and released in 2.6.0, but 2.8.0 added the need to merge overrides and + # that breaks this use case again. + for file in glob.glob(os.path.join(path, "*.yaml")): + index = Modulemd.ModuleIndex() + index.update_from_file(file, strict=False) + for module_name in index.get_module_names(): + yield module_name, index.get_module(module_name).get_defaults() + + +def collect_module_defaults( + defaults_dir, modules_to_load=None, mod_index=None, overrides_dir=None +): + """Load module defaults into index. + + If `modules_to_load` is passed in, it should be a set of module names. Only + defaults for these modules will be loaded. + + If `mod_index` is passed in, it will be updated and returned. If it was + not, a new ModuleIndex will be created and returned + """ + mod_index = mod_index or Modulemd.ModuleIndex() + + temp_index = Modulemd.ModuleIndex.new() + temp_index.update_from_defaults_directory( + defaults_dir, overrides_path=overrides_dir, strict=False + ) + + for module_name in temp_index.get_module_names(): + defaults = temp_index.get_module(module_name).get_defaults() + + if not modules_to_load or module_name in modules_to_load: + mod_index.add_defaults(defaults) + + return mod_index diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py index b2c67ad7..e0a1da9c 100644 --- a/pungi/phases/createrepo.py +++ b/pungi/phases/createrepo.py @@ -33,13 +33,8 @@ from kobo.shortcuts import run, relative_path from ..wrappers.scm import get_dir_from_scm from ..wrappers.createrepo import CreaterepoWrapper from .base import PhaseBase -from ..util import ( - find_old_compose, - get_arch_variant_data, - collect_module_defaults, - temp_dir, -) -from pungi import Modulemd +from ..util import find_old_compose, get_arch_variant_data, temp_dir +from ..module_util import Modulemd, collect_module_defaults import productmd.rpms import productmd.modules diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py index 736c10ec..2deda172 100644 --- a/pungi/phases/gather/__init__.py +++ b/pungi/phases/gather/__init__.py @@ -27,12 +27,12 @@ from .link import link_files from ...wrappers.createrepo import CreaterepoWrapper import pungi.wrappers.kojiwrapper -from pungi import Modulemd from pungi.compose import get_ordered_variant_uids from pungi.arch import get_compatible_arches, split_name_arch from pungi.phases.base import PhaseBase from pungi.util import (get_arch_data, get_arch_variant_data, get_variant_data, - makedirs, collect_module_defaults) + makedirs) +from pungi.module_util import Modulemd, collect_module_defaults from pungi.phases.createrepo import add_modular_metadata diff --git a/pungi/phases/gather/methods/method_hybrid.py b/pungi/phases/gather/methods/method_hybrid.py index ade3c4e8..448fd267 100644 --- a/pungi/phases/gather/methods/method_hybrid.py +++ b/pungi/phases/gather/methods/method_hybrid.py @@ -23,13 +23,11 @@ import kobo.rpmlib from kobo.shortcuts import run import pungi.phases.gather.method -from pungi import Modulemd, multilib_dnf +from pungi import multilib_dnf +from pungi.module_util import Modulemd from pungi.arch import get_valid_arches, tree_arch_to_yum_arch from pungi.phases.gather import _mk_pkg_map -from pungi.util import ( - get_arch_variant_data, - pkg_is_debug, -) +from pungi.util import get_arch_variant_data, pkg_is_debug from pungi.wrappers import fus from pungi.wrappers.comps import CompsWrapper diff --git a/pungi/phases/init.py b/pungi/phases/init.py index 3d5472fa..12969544 100644 --- a/pungi/phases/init.py +++ b/pungi/phases/init.py @@ -23,7 +23,8 @@ from kobo.threads import run_in_threads from pungi.phases.base import PhaseBase from pungi.phases.gather import write_prepopulate_file -from pungi.util import temp_dir, iter_module_defaults +from pungi.util import temp_dir +from pungi.module_util import iter_module_defaults from pungi.wrappers.comps import CompsWrapper from pungi.wrappers.createrepo import CreaterepoWrapper from pungi.wrappers.scm import get_dir_from_scm, get_file_from_scm diff --git a/pungi/phases/pkgset/common.py b/pungi/phases/pkgset/common.py index d7e06cfc..6e53cfc9 100644 --- a/pungi/phases/pkgset/common.py +++ b/pungi/phases/pkgset/common.py @@ -20,10 +20,10 @@ import threading from kobo.shortcuts import run, relative_path from kobo.threads import run_in_threads -from pungi import Modulemd from pungi.arch import get_valid_arches from pungi.wrappers.createrepo import CreaterepoWrapper -from pungi.util import is_arch_multilib, find_old_compose, collect_module_defaults +from pungi.util import is_arch_multilib, find_old_compose +from pungi.module_util import Modulemd, collect_module_defaults from pungi.phases.createrepo import add_modular_metadata diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index 3a0dfb52..187e6f1b 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -28,7 +28,7 @@ from pungi.wrappers.comps import CompsWrapper import pungi.phases.pkgset.pkgsets from pungi.arch import getBaseArch from pungi.util import retry, find_old_compose, get_arch_variant_data -from pungi import Modulemd +from pungi.module_util import Modulemd from pungi.phases.pkgset.common import MaterializedPackageSet, get_all_arches from pungi.phases.gather import get_packages_to_gather diff --git a/pungi/util.py b/pungi/util.py index bd3e022c..8bafec48 100644 --- a/pungi/util.py +++ b/pungi/util.py @@ -15,7 +15,6 @@ import argparse import fnmatch -import glob import json import subprocess import os @@ -36,8 +35,6 @@ import kobo.conf from kobo.shortcuts import run, force_list from productmd.common import get_major_version -from pungi import Modulemd - # Patterns that match all names of debuginfo packages DEBUG_PATTERNS = ["*-debuginfo", "*-debuginfo-*", "*-debugsource"] @@ -926,52 +923,6 @@ def parse_koji_event(event): ) -def iter_module_defaults(path): - """Given a path to a directory with yaml files, yield each module default - in there as a pair (module_name, ModuleDefaults instance). - """ - # It is really tempting to merge all the module indexes into a single one - # and work with it. However that does not allow for detecting conflicting - # defaults. That should not happen in practice, but better safe than sorry. - # Once libmodulemd can report the error, this code can be simplifed by a - # lot. It was implemented in - # https://github.com/fedora-modularity/libmodulemd/commit/3087e4a5c38a331041fec9b6b8f1a372f9ffe64d - # and released in 2.6.0, but 2.8.0 added the need to merge overrides and - # that breaks this use case again. - for file in glob.glob(os.path.join(path, "*.yaml")): - index = Modulemd.ModuleIndex() - index.update_from_file(file, strict=False) - for module_name in index.get_module_names(): - yield module_name, index.get_module(module_name).get_defaults() - - -def collect_module_defaults( - defaults_dir, modules_to_load=None, mod_index=None, overrides_dir=None -): - """Load module defaults into index. - - If `modules_to_load` is passed in, it should be a set of module names. Only - defaults for these modules will be loaded. - - If `mod_index` is passed in, it will be updated and returned. If it was - not, a new ModuleIndex will be created and returned - """ - mod_index = mod_index or Modulemd.ModuleIndex() - - temp_index = Modulemd.ModuleIndex.new() - temp_index.update_from_defaults_directory( - defaults_dir, overrides_path=overrides_dir, strict=False - ) - - for module_name in temp_index.get_module_names(): - defaults = temp_index.get_module(module_name).get_defaults() - - if not modules_to_load or module_name in modules_to_load: - mod_index.add_defaults(defaults) - - return mod_index - - def load_config(file_path, defaults={}): """Open and load configuration file form .conf or .json file.""" conf = kobo.conf.PyConfigParser() diff --git a/tests/helpers.py b/tests/helpers.py index ea1b4936..9650d401 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -18,7 +18,8 @@ except ImportError: import unittest from pungi.util import get_arch_variant_data -from pungi import paths, checks, Modulemd +from pungi import paths, checks +from pungi.module_util import Modulemd class BaseTestCase(unittest.TestCase): diff --git a/tests/test_createrepophase.py b/tests/test_createrepophase.py index cc3e9488..c7c43c5d 100644 --- a/tests/test_createrepophase.py +++ b/tests/test_createrepophase.py @@ -19,7 +19,7 @@ from pungi.phases.createrepo import (CreaterepoPhase, get_productids_from_scm, ModulesMetadata) from tests.helpers import DummyCompose, PungiTestCase, copy_fixture, touch -from pungi import Modulemd +from pungi.module_util import Modulemd class TestCreaterepoPhase(PungiTestCase): diff --git a/tests/test_gather_source_module.py b/tests/test_gather_source_module.py index 400cc9c5..9446bc80 100644 --- a/tests/test_gather_source_module.py +++ b/tests/test_gather_source_module.py @@ -13,7 +13,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) from pungi.phases.gather.sources.source_module import GatherSourceModule from tests import helpers -from pungi import Modulemd +from pungi.module_util import Modulemd @unittest.skipUnless(Modulemd is not None, "Skipped test, no module support.") diff --git a/tests/test_initphase.py b/tests/test_initphase.py index 4783e230..964913d6 100644 --- a/tests/test_initphase.py +++ b/tests/test_initphase.py @@ -13,7 +13,7 @@ import sys sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) -from pungi import Modulemd +from pungi.module_util import Modulemd from pungi.phases import init from tests.helpers import DummyCompose, PungiTestCase, touch, mk_boom, fake_run_in_threads diff --git a/tests/test_pkgset_common.py b/tests/test_pkgset_common.py index d1e147fe..03321a9c 100755 --- a/tests/test_pkgset_common.py +++ b/tests/test_pkgset_common.py @@ -8,7 +8,7 @@ import mock sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) -from pungi import Modulemd +from pungi.module_util import Modulemd from pungi.phases.pkgset import common from tests import helpers diff --git a/tests/test_pkgset_source_koji.py b/tests/test_pkgset_source_koji.py index bd939683..a9d7fb3d 100644 --- a/tests/test_pkgset_source_koji.py +++ b/tests/test_pkgset_source_koji.py @@ -14,7 +14,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) from pungi.phases.pkgset.sources import source_koji from tests import helpers -from pungi import Modulemd +from pungi.module_util import Modulemd EVENT_INFO = {'id': 15681980, 'ts': 1460956382.81936} TAG_INFO = { @@ -613,7 +613,7 @@ class MockModule(object): return self.path == other.path -@mock.patch("pungi.Modulemd.ModuleStream.read_file", new=MockModule) +@mock.patch("pungi.module_util.Modulemd.ModuleStream.read_file", new=MockModule) @unittest.skipIf(Modulemd is None, "Skipping tests, no module support") class TestAddModuleToVariant(helpers.PungiTestCase):