pungi/phases/pkgset/pkgsets.py

@@ -889,6 +889,8 @@ class KojiMockPackageSet(KojiPackageSet):
             sigkey.lower() for sigkey in self.sigkey_ordering
             if sigkey is not None
         ]
+        if not sigkeys:
+            return True
         with open(rpm_path, 'rb') as fd:
             header = ts.hdrFromFdno(fd)
             signature = header[rpm.RPMTAG_SIGGPG] or header[rpm.RPMTAG_SIGPGP]

pungi/phases/pkgset/sources/source_kojimock.py

@@ -35,10 +35,12 @@ import pungi.wrappers.kojiwrapper
 from pungi.wrappers.comps import CompsWrapper
 from pungi.wrappers.mbs import MBSWrapper
 import pungi.phases.pkgset.pkgsets
+
 from pungi.util import (
     retry,
     get_arch_variant_data,
     get_variant_data,
+
     read_single_module_stream_from_string,
     read_single_module_stream_from_file,
 )
@@ -160,14 +162,16 @@ def get_koji_modules(compose, koji_wrapper, event, module_info_str):
             # Store module versioning information into the dict, but make sure
             # not to overwrite any existing keys.
             md["module_stream"] = md["extra"]["typeinfo"]["module"]["stream"]
-            md["module_version"] = int(md["extra"]["typeinfo"]["module"]["version"])
+            md["module_version"] = int(
+                md["extra"]["typeinfo"]["module"]["version"])
             md["module_context"] = md["extra"]["typeinfo"]["module"]["context"]
         except KeyError:
             continue
 
         if md["state"] == pungi.wrappers.kojiwrapper.KOJI_BUILD_DELETED:
             compose.log_debug(
-                "Module build %s has been deleted, ignoring it." % build["name"]
+                "Module build %s has been deleted, ignoring it." % build[
+                    "name"]
             )
             continue
 
@@ -189,7 +193,8 @@ def get_koji_modules(compose, koji_wrapper, event, module_info_str):
         )
         latest_version = sorted_modules[0]["module_version"]
         modules = [
-            module for module in modules if latest_version == module["module_version"]
+            module for module in modules
+            if latest_version == module["module_version"]
         ]
 
     return modules
@@ -205,7 +210,8 @@ class PkgsetSourceKojiMock(pungi.phases.pkgset.source.PkgsetSourceBase):
             get_all_arches(compose),
         )
         # path prefix must contain trailing '/'
-        path_prefix = self.koji_wrapper.koji_module.config.topdir.rstrip("/") + "/"
+        path_prefix = self.koji_wrapper.koji_module.config.topdir.rstrip(
+            "/") + "/"
         package_sets = get_pkgset_from_koji(
             self.compose, self.koji_wrapper, path_prefix
         )
@@ -214,7 +220,8 @@ class PkgsetSourceKojiMock(pungi.phases.pkgset.source.PkgsetSourceBase):
 
 def get_pkgset_from_koji(compose, koji_wrapper, path_prefix):
     event_info = get_koji_event_info(compose, koji_wrapper)
-    return populate_global_pkgset(compose, koji_wrapper, path_prefix, event_info)
+    return populate_global_pkgset(compose, koji_wrapper, path_prefix,
+                                  event_info)
 
 
 def _add_module_to_variant(
@@ -241,13 +248,16 @@ def _add_module_to_variant(
         if archive["btype"] != "module":
             # Skip non module archives
             continue
+
         filename = archive["filename"]
         file_path = os.path.join(
             koji_wrapper.koji_module.pathinfo.topdir,
             'modules',
             build['arch'],
             build['extra']['typeinfo']['module']['content_koji_tag']
+
         )
+
         mmds[filename] = file_path
 
     if len(mmds) <= 1:
@@ -266,17 +276,22 @@ def _add_module_to_variant(
     added = False
 
     for arch in variant.arches:
-        if _is_filtered_out(compose, variant, arch, info["name"], info["stream"]):
-            compose.log_debug("Module %s is filtered from %s.%s", nsvc, variant, arch)
+        if _is_filtered_out(compose, variant, arch, info["name"],
+                            info["stream"]):
+            compose.log_debug("Module %s is filtered from %s.%s", nsvc,
+                              variant, arch)
             continue
+
         filename = "modulemd.%s.txt" % arch
         try:
             mod_stream = read_single_module_stream_from_file(
                 mmds[filename], compose, arch, build
+
             )
             if mod_stream:
                 added = True
             variant.arch_mmds.setdefault(arch, {})[nsvc] = mod_stream
+
             added = True
         except KeyError:
             # There is no modulemd for this arch. This could mean an arch was
@@ -298,7 +313,8 @@ def _add_extra_modules_to_variant(
         compose, koji_wrapper, variant, extra_modules, variant_tags, tag_to_mmd
 ):
     for nsvc in extra_modules:
-        msg = "Adding extra module build '%s' to variant '%s'" % (nsvc, variant)
+        msg = "Adding extra module build '%s' to variant '%s'" % (
+        nsvc, variant)
         compose.log_info(msg)
 
         nsvc_info = nsvc.split(":")
@@ -344,7 +360,8 @@ def _add_scratch_modules_to_variant(
         compose, variant, scratch_modules, variant_tags, tag_to_mmd
 ):
     if compose.compose_type != "test" and scratch_modules:
-        compose.log_warning("Only test composes could include scratch module builds")
+        compose.log_warning(
+            "Only test composes could include scratch module builds")
         return
 
     mbs = MBSWrapper(compose.conf["mbs_api_url"])
@@ -355,7 +372,8 @@ def _add_scratch_modules_to_variant(
         try:
             final_modulemd = mbs.final_modulemd(module_build["id"])
         except Exception:
-            compose.log_error("Unable to get modulemd for build %s" % module_build)
+            compose.log_error(
+                "Unable to get modulemd for build %s" % module_build)
             raise
         tag = module_build["koji_tag"]
         variant_tags[variant].append(tag)
@@ -363,8 +381,7 @@ def _add_scratch_modules_to_variant(
         for arch in variant.arches:
             try:
                 mmd = read_single_module_stream_from_string(
-                    final_modulemd[arch]
-                )
+                    final_modulemd[arch])
                 variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd
             except KeyError:
                 continue
@@ -390,21 +407,24 @@ def _is_filtered_out(compose, variant, arch, module_name, module_stream):
     if not compose:
         return False
 
-    for filter in get_arch_variant_data(compose.conf, "filter_modules", arch, variant):
+    for filter in get_arch_variant_data(compose.conf, "filter_modules", arch,
+                                        variant):
         if ":" not in filter:
             name_filter = filter
             stream_filter = "*"
         else:
             name_filter, stream_filter = filter.split(":", 1)
 
-        if fnmatch(module_name, name_filter) and fnmatch(module_stream, stream_filter):
+        if fnmatch(module_name, name_filter) and fnmatch(module_stream,
+                                                         stream_filter):
             return True
 
     return False
 
 
 def _get_modules_from_koji(
-    compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd
+        compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd,
+        exclude_module_ns
 ):
     """
     Loads modules for given `variant` from koji `session`, adds them to
@@ -415,15 +435,21 @@ def _get_modules_from_koji(
     :param Variant variant: Variant with modules to find.
     :param dict variant_tags: Dict populated by this method. Key is `variant`
         and value is list of Koji tags to get the RPMs from.
+    :param list exclude_module_ns: Module name:stream which will be excluded.
     """
 
     # Find out all modules in every variant and add their Koji tags
     # to variant and variant_tags list.
     for module in variant.get_modules():
-        koji_modules = get_koji_modules(compose, koji_wrapper, event, module["name"])
+        koji_modules = get_koji_modules(compose, koji_wrapper, event,
+                                        module["name"])
         for koji_module in koji_modules:
             nsvc = _add_module_to_variant(
-                koji_wrapper, variant, koji_module, compose=compose
+                koji_wrapper,
+                variant,
+                koji_module,
+                compose=compose,
+                exclude_module_ns=exclude_module_ns,
             )
             if not nsvc:
                 continue
@@ -462,7 +488,8 @@ def filter_inherited(koji_proxy, event, module_builds, top_tag):
     does not understand streams, so we have to reimplement it here.
     """
     inheritance = [
-        tag["name"] for tag in koji_proxy.getFullInheritance(top_tag, event=event["id"])
+        tag["name"] for tag in
+        koji_proxy.getFullInheritance(top_tag, event=event["id"])
     ]
 
     def keyfunc(mb):
@@ -487,7 +514,8 @@ def filter_inherited(koji_proxy, event, module_builds, top_tag):
     return result
 
 
-def filter_by_whitelist(compose, module_builds, input_modules, expected_modules):
+def filter_by_whitelist(compose, module_builds, input_modules,
+                        expected_modules):
     """
     Exclude modules from the list that do not match any pattern specified in
     input_modules. Order may not be preserved. The last argument is a set of
@@ -511,6 +539,7 @@ def filter_by_whitelist(compose, module_builds, input_modules, expected_modules)
             info.get("context"),
         )
         nvr_patterns.add((pattern, spec["name"]))
+
     modules_to_keep = []
 
     for mb in sorted(module_builds, key=lambda i: i['name']):
@@ -575,7 +604,13 @@ def _filter_expected_modules(
 
 
 def _get_modules_from_koji_tags(
-    compose, koji_wrapper, event_id, variant, variant_tags, tag_to_mmd
+        compose,
+        koji_wrapper,
+        event_id,
+        variant,
+        variant_tags,
+        tag_to_mmd,
+        exclude_module_ns,
 ):
     """
     Loads modules for given `variant` from Koji, adds them to
@@ -587,10 +622,12 @@ def _get_modules_from_koji_tags(
     :param Variant variant: Variant with modules to find.
     :param dict variant_tags: Dict populated by this method. Key is `variant`
         and value is list of Koji tags to get the RPMs from.
+    :param list exclude_module_ns: Module name:stream which will be excluded.
     """
     # Compose tags from configuration
     compose_tags = [
-        {"name": tag} for tag in force_list(compose.conf["pkgset_koji_module_tag"])
+        {"name": tag} for tag in
+        force_list(compose.conf["pkgset_koji_module_tag"])
     ]
     # Get set of configured module names for this variant. If nothing is
     # configured, the set is empty.
@@ -617,7 +654,8 @@ def _get_modules_from_koji_tags(
         )
 
         # Filter out builds inherited from non-top tag
-        module_builds = filter_inherited(koji_proxy, event_id, module_builds, tag)
+        module_builds = filter_inherited(koji_proxy, event_id, module_builds,
+                                         tag)
 
         # Apply whitelist of modules if specified.
         variant_modules = variant.get_modules()
@@ -625,6 +663,7 @@ def _get_modules_from_koji_tags(
             module_builds = filter_by_whitelist(
                 compose, module_builds, variant_modules, expected_modules
             )
+
         # Find the latest builds of all modules. This does following:
         # - Sorts the module_builds descending by Koji NVR (which maps to NSV
         #   for modules). Split release into modular version and context, and
@@ -662,6 +701,18 @@ def _get_modules_from_koji_tags(
         for build in latest_builds:
             # Get the Build from Koji to get modulemd and module_tag.
             build = koji_proxy.getBuild(build["build_id"])
+
+            nsvc = _add_module_to_variant(
+                koji_wrapper,
+                variant,
+                build,
+                True,
+                compose=compose,
+                exclude_module_ns=exclude_module_ns,
+            )
+            if not nsvc:
+                continue
+
             module_tag = (
                 build.get("extra", {})
                 .get("typeinfo", {})
@@ -671,12 +722,6 @@ def _get_modules_from_koji_tags(
 
             variant_tags[variant].append(module_tag)
 
-            nsvc = _add_module_to_variant(
-                koji_wrapper, variant, build, True, compose=compose
-            )
-            if not nsvc:
-                continue
-
             tag_to_mmd.setdefault(module_tag, {})
             for arch in variant.arch_mmds:
                 try:
@@ -767,26 +812,48 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
                 "modules."
             )
 
+        extra_modules = get_variant_data(
+            compose.conf, "pkgset_koji_module_builds", variant
+        )
+
+        # When adding extra modules, other modules of the same name:stream available
+        # in brew tag should be excluded.
+        exclude_module_ns = []
+        if extra_modules:
+            exclude_module_ns = [
+                ":".join(nsvc.split(":")[:2]) for nsvc in extra_modules
+            ]
+
         if modular_koji_tags or (
                 compose.conf["pkgset_koji_module_tag"] and variant.modules
         ):
             # List modules tagged in particular tags.
             _get_modules_from_koji_tags(
-                compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd
+                compose,
+                koji_wrapper,
+                event,
+                variant,
+                variant_tags,
+                tag_to_mmd,
+                exclude_module_ns,
             )
         elif variant.modules:
             # Search each module in Koji separately. Tagging does not come into
             # play here.
             _get_modules_from_koji(
-                compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd
+                compose,
+                koji_wrapper,
+                event,
+                variant,
+                variant_tags,
+                tag_to_mmd,
+                exclude_module_ns,
             )
 
-        extra_modules = get_variant_data(
-            compose.conf, "pkgset_koji_module_builds", variant
-        )
         if extra_modules:
             _add_extra_modules_to_variant(
-                compose, koji_wrapper, variant, extra_modules, variant_tags, tag_to_mmd
+                compose, koji_wrapper, variant, extra_modules, variant_tags,
+                tag_to_mmd
             )
 
         variant_scratch_modules = get_variant_data(
@@ -794,7 +861,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
         )
         if variant_scratch_modules:
             _add_scratch_modules_to_variant(
-                compose, variant, variant_scratch_modules, variant_tags, tag_to_mmd
+                compose, variant, variant_scratch_modules, variant_tags,
+                tag_to_mmd
             )
 
         # Ensure that every tag added to `variant_tags` is added also to
@@ -819,8 +887,10 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
     for compose_tag in compose_tags:
         compose.log_info("Loading package set for tag %s", compose_tag)
         if compose_tag in pkgset_koji_tags:
-            extra_builds = force_list(compose.conf.get("pkgset_koji_builds", []))
-            extra_tasks = force_list(compose.conf.get("pkgset_koji_scratch_tasks", []))
+            extra_builds = force_list(
+                compose.conf.get("pkgset_koji_builds", []))
+            extra_tasks = force_list(
+                compose.conf.get("pkgset_koji_scratch_tasks", []))
         else:
             extra_builds = []
             extra_tasks = []
@@ -926,7 +996,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
 
 
 def get_koji_event_info(compose, koji_wrapper):
-    event_file = os.path.join(compose.paths.work.topdir(arch="global"), "koji-event")
+    event_file = os.path.join(compose.paths.work.topdir(arch="global"),
+                              "koji-event")
 
     compose.log_info("Getting koji event")
     result = get_koji_event_raw(koji_wrapper, compose.koji_event, event_file)

pungi/scripts/create_packages_json.py

@@ -14,7 +14,7 @@ import os
 import re
 import tempfile
 from collections import defaultdict
-from typing import AnyStr, Dict, List, Optional, Any, Iterator
+from typing import AnyStr, Dict, List, Any, Iterator
 
 import binascii
 import createrepo_c as cr
@@ -65,7 +65,7 @@ class RepoInfo:
     # Only layout of specific package (which don't exist
     # in a reference repository) will be taken as example
     is_reference: bool = False
-    strict_arch: bool = False
+    repo_type: str = 'present'
 
 
 class PackagesGenerator:
@@ -262,7 +262,11 @@ class PackagesGenerator:
             )
         )
         all_packages = defaultdict(lambda: {'variants': list()})
-        for repo_info in self.repos:
+        for repo_info in sorted(
+                self.repos,
+                key=lambda i: i.repo_type,
+                reverse=True,
+        ):
             repomd_records = self._get_repomd_records(
                 repo_info=repo_info,
             )
@@ -298,6 +302,8 @@ class PackagesGenerator:
                     all_packages[package_key]['arch'] = package_arch
                     all_packages[package_key]['package'] = package
                     all_packages[package_key]['type'] = repo_info.is_reference
+                elif repo_info.repo_type == 'absent' and (repo_info.name, repo_info.arch) in all_packages[package_key]['variants']:
+                    all_packages[package_key]['variants'].remove((repo_info.name, repo_info.arch))
                 # replace an older package if it's not reference or
                 # a newer package is from reference repo
                 elif (not all_packages[package_key]['type'] or
@@ -404,6 +410,14 @@ def create_parser():
         choices=['yes', 'no'],
         required=True,
     )
+    parser.add_argument(
+        '--repo-type',
+        action='append',
+        type=str,
+        help='Packages from repository will be removed or added to variant',
+        choices=['present', 'absent'],
+        required=True,
+    )
     parser.add_argument(
         '--excluded-packages',
         nargs='+',
@@ -436,13 +450,14 @@ def cli_main():
     args = create_parser().parse_args()
     repos = []
     for repo_path, repo_folder, repo_name, \
-        repo_arch, is_remote, is_reference in zip(
+        repo_arch, is_remote, is_reference, repo_type in zip(
             args.repo_path,
             args.repo_folder,
             args.repo_name,
             args.repo_arch,
             args.is_remote,
             args.is_reference,
+            args.repo_type,
             ):
         repos.append(RepoInfo(
             path=repo_path,
@@ -450,7 +465,8 @@ def cli_main():
             name=repo_name,
             arch=repo_arch,
             is_remote=True if is_remote == 'yes' else False,
-            is_reference=True if is_reference == 'yes' else False
+            is_reference=True if is_reference == 'yes' else False,
+            repo_type=repo_type,
         ))
     pg = PackagesGenerator(
         repos=repos,