- Fix unittests

pungi/phases/pkgset/pkgsets.py

@@ -828,6 +828,8 @@ class KojiMockPackageSet(PackageSetBase):
         cache_region=None,
         extra_builds=None,
         extra_tasks=None,
+        signed_packages_retries=0,
+        signed_packages_wait=30,
     ):
         """
         Creates new KojiPackageSet.
@@ -876,10 +878,11 @@ class KojiMockPackageSet(PackageSetBase):
         self.extra_builds = extra_builds or []
         self.extra_tasks = extra_tasks or []
         self.reuse = None
+        self.signed_packages_retries = signed_packages_retries
+        self.signed_packages_wait = signed_packages_wait
 
     def __getstate__(self):
         result = self.__dict__.copy()
-        result["koji_profile"] = self.koji_wrapper.profile
         del result["koji_wrapper"]
         del result["_logger"]
         if "cache_region" in result:
@@ -887,8 +890,6 @@ class KojiMockPackageSet(PackageSetBase):
         return result
 
     def __setstate__(self, data):
-        koji_profile = data.pop("koji_profile")
-        self.koji_wrapper = pungi.wrappers.kojiwrapper.KojiWrapper(koji_profile)
         self._logger = None
         self.__dict__.update(data)
 
@@ -1202,20 +1203,26 @@ class KojiMockPackageSet(PackageSetBase):
                 % (old_koji_event, koji_event)
             )
             changed = self.koji_proxy.queryHistory(
-                tables=["tag_listing"], tag=tag, afterEvent=old_koji_event
+                tables=["tag_listing", "tag_inheritance"],
+                tag=tag,
+                afterEvent=min(koji_event, old_koji_event),
+                beforeEvent=max(koji_event, old_koji_event) + 1,
             )
             if changed["tag_listing"]:
                 self.log_debug("Builds under tag %s changed. Can't reuse." % tag)
                 return False
+            if changed["tag_inheritance"]:
+                self.log_debug("Tag inheritance %s changed. Can't reuse." % tag)
+                return False
 
             if inherit:
                 inherit_tags = self.koji_proxy.getFullInheritance(tag, koji_event)
                 for t in inherit_tags:
                     changed = self.koji_proxy.queryHistory(
-                        tables=["tag_listing"],
+                        tables=["tag_listing", "tag_inheritance"],
                         tag=t["name"],
-                        afterEvent=old_koji_event,
+                        afterEvent=min(koji_event, old_koji_event),
-                        beforeEvent=koji_event + 1,
+                        beforeEvent=max(koji_event, old_koji_event) + 1,
                     )
                     if changed["tag_listing"]:
                         self.log_debug(
@@ -1223,6 +1230,9 @@ class KojiMockPackageSet(PackageSetBase):
                             % t["name"]
                         )
                         return False
+                    if changed["tag_inheritance"]:
+                        self.log_debug("Tag inheritance %s changed. Can't reuse." % tag)
+                        return False
 
         repo_dir = compose.paths.work.pkgset_repo(tag, create_dir=False)
         old_repo_dir = compose.paths.old_compose_path(repo_dir)

pungi/phases/pkgset/sources/source_kojimock.py

@@ -35,7 +35,13 @@ import pungi.wrappers.kojiwrapper
 from pungi.wrappers.comps import CompsWrapper
 from pungi.wrappers.mbs import MBSWrapper
 import pungi.phases.pkgset.pkgsets
-from pungi.util import retry, get_arch_variant_data, get_variant_data
+from pungi.util import (
+    retry,
+    get_arch_variant_data,
+    get_variant_data,
+    read_single_module_stream_from_string,
+    read_single_module_stream_from_file,
+)
 from pungi.module_util import Modulemd
 
 from pungi.phases.pkgset.common import MaterializedPackageSet, get_all_arches
@@ -212,7 +218,12 @@ def get_pkgset_from_koji(compose, koji_wrapper, path_prefix):
 
 
 def _add_module_to_variant(
-    koji_wrapper, variant, build, add_to_variant_modules=False, compose=None
+    koji_wrapper,
+    variant,
+    build,
+    add_to_variant_modules=False,
+    compose=None,
+    exclude_module_ns=None,
 ):
     """
     Adds module defined by Koji build info to variant.
@@ -222,6 +233,7 @@ def _add_module_to_variant(
     :param bool add_to_variant_modules: Adds the modules also to
         variant.modules.
     :param compose: Compose object to get filters from
+    :param list exclude_module_ns: Module name:stream which will be excluded.
     """
     mmds = {}
     archives = koji_wrapper.koji_proxy.listArchives(build["id"])
@@ -246,6 +258,10 @@ def _add_module_to_variant(
 
     info = build["extra"]["typeinfo"]["module"]
     nsvc = "%(name)s:%(stream)s:%(version)s:%(context)s" % info
+    ns = "%(name)s:%(stream)s" % info
+
+    if exclude_module_ns and ns in exclude_module_ns:
+        return
 
     added = False
 
@@ -253,12 +269,14 @@ def _add_module_to_variant(
         if _is_filtered_out(compose, variant, arch, info["name"], info["stream"]):
             compose.log_debug("Module %s is filtered from %s.%s", nsvc, variant, arch)
             continue
-
+        filename = "modulemd.%s.txt" % arch
         try:
-            mmd = Modulemd.ModuleStream.read_file(
+            mod_stream = read_single_module_stream_from_file(
-                mmds["modulemd.%s.txt" % arch], strict=True
+                mmds[filename], compose, arch, build
             )
-            variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd
+            if mod_stream:
+                added = True
+            variant.arch_mmds.setdefault(arch, {})[nsvc] = mod_stream
             added = True
         except KeyError:
             # There is no modulemd for this arch. This could mean an arch was
@@ -344,8 +362,8 @@ def _add_scratch_modules_to_variant(
         tag_to_mmd.setdefault(tag, {})
         for arch in variant.arches:
             try:
-                mmd = Modulemd.ModuleStream.read_string(
+                mmd = read_single_module_stream_from_string(
-                    final_modulemd[arch], strict=True
+                    final_modulemd[arch]
                 )
                 variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd
             except KeyError:

tests/fixtures/mmds/modules/fake_arch/module:master-20190318-abcdef

@@ -0,0 +1,20 @@
+---
+document: modulemd
+version: 2
+data:
+  name: module
+  stream: master
+  version: 20190318
+  context: abcdef
+  arch: armhfp
+  summary: Dummy module
+  description: Dummy module
+  license:
+    module:
+    - Beerware
+    content:
+    - Beerware
+  artifacts:
+    rpms:
+      - foobar-0:1.0-1.noarch
+...

tests/test_pkgset_common.py

@@ -129,16 +129,16 @@ class TestMaterializedPkgsetCreate(helpers.PungiTestCase):
         amm.assert_called_once()
 
         self.assertEqual(
-            amm.mock_calls[0].args[1], os.path.join(self.topdir, "work/x86_64/repo/foo")
+            amm.mock_calls[0][1][1], os.path.join(self.topdir, "work/x86_64/repo/foo")
         )
-        self.assertIsInstance(amm.mock_calls[0].args[2], Modulemd.ModuleIndex)
+        self.assertIsInstance(amm.mock_calls[0][1][2], Modulemd.ModuleIndex)
-        self.assertIsNotNone(amm.mock_calls[0].args[2].get_module("mod_name"))
+        self.assertIsNotNone(amm.mock_calls[0][1][2].get_module("mod_name"))
         # Check if proper Index is used by add_modular_metadata
         self.assertIsNotNone(
-            amm.mock_calls[0].args[2].get_module("mod_name").get_obsoletes()
+            amm.mock_calls[0][1][2].get_module("mod_name").get_obsoletes()
         )
         self.assertEqual(
-            amm.mock_calls[0].args[3],
+            amm.mock_calls[0][1][3],
             os.path.join(self.topdir, "logs/x86_64/arch_repo_modulemd.foo.x86_64.log"),
         )
 

tests/test_pkgset_pkgsets.py

@@ -312,13 +312,9 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
         )
         self.assertRegex(str(ctx.exception), figure)
 
-    @ddt.data(
-        pkgsets.KojiPackageSet,
-        pkgsets.KojiMockPackageSet,
-    )
     @mock.patch("os.path.isfile")
     @mock.patch("time.sleep")
-    def test_find_signed_after_wait(self, sleep, isfile, package_set):
+    def test_find_signed_after_wait(self, sleep, isfile):
         checked_files = set()
 
         def check_file(path):
@@ -331,7 +327,7 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
         isfile.side_effect = check_file
 
         fst_key, snd_key = ["cafebabe", "deadbeef"]
-        pkgset = package_set(
+        pkgset = pkgsets.KojiPackageSet(
             "pkgset",
             self.koji_wrapper,
             [fst_key, snd_key],
@@ -411,13 +407,9 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
             r"^RPM\(s\) not found for sigs: .+Check log for details.+",
         )
 
-    @ddt.data(
-        pkgsets.KojiPackageSet,
-        pkgsets.KojiMockPackageSet,
-    )
     @mock.patch("time.sleep")
-    def test_can_not_find_signed_package_with_retries(self, time, package_set):
+    def test_can_not_find_signed_package_with_retries(self, time):
-        pkgset = package_set(
+        pkgset = pkgsets.KojiPackageSet(
             "pkgset",
             self.koji_wrapper,
             ["cafebabe"],
@@ -767,8 +759,8 @@ class TestReuseKojiPkgset(helpers.PungiTestCase):
         mock_old_topdir.return_value = self.old_compose_dir
         self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1])
         self.koji_wrapper.koji_proxy.queryHistory.side_effect = [
-            {"tag_listing": []},
+            {"tag_listing": [], "tag_inheritance": []},
-            {"tag_listing": [{}]},
+            {"tag_listing": [{}], "tag_inheritance": []},
         ]
         self.koji_wrapper.koji_proxy.getFullInheritance.return_value = [
             {"name": self.inherited_tag}
@@ -795,7 +787,9 @@ class TestReuseKojiPkgset(helpers.PungiTestCase):
     def test_reuse_failed_load_reuse_file(self, mock_old_topdir, mock_exists):
         mock_old_topdir.return_value = self.old_compose_dir
         self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1])
-        self.koji_wrapper.koji_proxy.queryHistory.return_value = {"tag_listing": []}
+        self.koji_wrapper.koji_proxy.queryHistory.return_value = {
+            "tag_listing": [], "tag_inheritance": []
+        }
         self.koji_wrapper.koji_proxy.getFullInheritance.return_value = []
         self.pkgset.load_old_file_cache = mock.Mock(
             side_effect=Exception("unknown error")
@@ -827,7 +821,9 @@ class TestReuseKojiPkgset(helpers.PungiTestCase):
     def test_reuse_criteria_not_match(self, mock_old_topdir, mock_exists):
         mock_old_topdir.return_value = self.old_compose_dir
         self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1])
-        self.koji_wrapper.koji_proxy.queryHistory.return_value = {"tag_listing": []}
+        self.koji_wrapper.koji_proxy.queryHistory.return_value = {
+            "tag_listing": [], "tag_inheritance": []
+        }
         self.koji_wrapper.koji_proxy.getFullInheritance.return_value = []
         self.pkgset.load_old_file_cache = mock.Mock(
             return_value={"allow_invalid_sigkeys": True}
@@ -866,7 +862,9 @@ class TestReuseKojiPkgset(helpers.PungiTestCase):
     def test_reuse_pkgset(self, mock_old_topdir, mock_exists, mock_copy_all):
         mock_old_topdir.return_value = self.old_compose_dir
         self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1])
-        self.koji_wrapper.koji_proxy.queryHistory.return_value = {"tag_listing": []}
+        self.koji_wrapper.koji_proxy.queryHistory.return_value = {
+            "tag_listing": [], "tag_inheritance": []
+        }
         self.koji_wrapper.koji_proxy.getFullInheritance.return_value = []
         self.pkgset.load_old_file_cache = mock.Mock(
             return_value={

tests/test_pkgset_source_koji.py

@@ -818,7 +818,7 @@ class TestAddModuleToVariantForKojiMock(helpers.PungiTestCase):
         super(TestAddModuleToVariantForKojiMock, self).setUp()
         self.koji = mock.Mock()
         self.koji.koji_module.pathinfo.typedir.return_value = "/koji"
-        self.koji.koji_module.pathinfo.topdir = "/mnt/koji"
+        self.koji.koji_module.pathinfo.topdir = MMDS_DIR
         files = ["modulemd.x86_64.txt", "modulemd.armv7hl.txt", "modulemd.txt"]
         self.koji.koji_proxy.listArchives.return_value = [
             {"btype": "module", "filename": fname} for fname in files