gather: Report missing comps packages

When a package mentioned in comps is not available in the package set,
print a warning about this. Additionally there is a config option that
allows to turn this warning into a fatal error.

Fixes: https://pagure.io/pungi/issue/50
Fixes: https://pagure.io/pungi/issue/683
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

doc/configuration.rst

@@ -627,6 +627,11 @@ Options
     (*bool*) -- Set to ``False`` if you don't want the compose to abort when
     some package has broken dependencies.
 
+**require_all_comps_packages** = False
+    (*bool*) -- Set to ``True`` to abort compose when package mentioned in
+    comps file can not be found in the package set. When disabled (the default,
+    such cases are still reported as warnings in the log.
+
 **gather_source_mapping**
     (*str*) -- Only use when ``gather_source = "json"``. The value should be a
     path to JSON file with following mapping: ``{variant: {arch: {rpm_name:

pungi/checks.py

@@ -575,6 +575,10 @@ def _make_schema():
                 "type": "boolean",
                 "default": True
             },
+            "require_all_comps_packages": {
+                "type": "boolean",
+                "default": False,
+            },
             "bootable": {
                 "type": "boolean",
                 "default": False

pungi/phases/gather/methods/method_deps.py

@@ -154,6 +154,15 @@ def resolve_deps(compose, arch, variant):
 
     with open(pungi_log, "r") as f:
         result = pungi_wrapper.get_packages(f)
+        f.seek(0)
+        missing_comps_pkgs = pungi_wrapper.get_missing_comps_packages(f)
+
+    if missing_comps_pkgs:
+        msg = ("Packages mentioned in comps do not exist for %s.%s: %s"
+               % (variant.uid, arch, ", ".join(sorted(missing_comps_pkgs))))
+        compose.log_warning(msg)
+        if compose.conf['require_all_comps_packages']:
+            raise RuntimeError(msg)
 
     compose.log_info("[DONE ] %s" % msg)
     return result

pungi/wrappers/pungi.py

@@ -29,6 +29,8 @@ PACKAGES_RE = {
 
 UNRESOLVED_DEPENDENCY_RE = re.compile(r"^.*Unresolvable dependency (.+) in ([^ ]+).*$")
 
+MISSING_COMPS_PACKAGE_RE = re.compile(r"^.*Could not find a match for (.+) in any configured repo")
+
 
 class PungiWrapper(object):
 
@@ -221,6 +223,14 @@ class PungiWrapper(object):
 
         return result
 
+    def get_missing_comps_packages(self, f):
+        result = set()
+        for line in f:
+            match = MISSING_COMPS_PACKAGE_RE.match(line)
+            if match:
+                result.add(match.group(1))
+        return result
+
     def get_missing_deps(self, f):
         result = {}