gather: Only parse pungi log once

No need to read the file three times, we can just get all the data in
one go.

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

contrib/yum-dnf-compare/pungi-compare-depsolving

@@ -19,7 +19,7 @@ from pungi.wrappers.pungi import PungiWrapper
 def read_rpms(fn):
     pw = PungiWrapper()
     with open(fn, "r") as f:
-        data = pw.get_packages(f)
+        data, _, _ = pw.parse_log(f)
     result = set()
     for i in data["rpm"]:
         nvra = parse_nvra(i["path"])

pungi/phases/gather/methods/method_deps.py

@@ -41,8 +41,8 @@ class GatherMethodDeps(pungi.phases.gather.method.GatherMethodBase):
         write_pungi_config(self.compose, arch, variant, packages, groups, filter_packages,
                            multilib_whitelist, multilib_blacklist,
                            fulltree_excludes=fulltree_excludes, prepopulate=prepopulate)
-        result = resolve_deps(self.compose, arch, variant)
+        result, missing_deps = resolve_deps(self.compose, arch, variant)
-        check_deps(self.compose, arch, variant)
+        check_deps(self.compose, arch, variant, missing_deps)
         return result
 
 
@@ -103,7 +103,8 @@ def resolve_deps(compose, arch, variant):
     if compose.DEBUG and os.path.exists(pungi_log):
         compose.log_warning("[SKIP ] %s" % msg)
         with open(pungi_log, "r") as f:
-            return pungi_wrapper.get_packages(f)
+            res, broken_deps, _ = pungi_wrapper.parse_log(f)
+            return res, broken_deps
 
     compose.log_info("[BEGIN] %s" % msg)
     pungi_conf = compose.paths.work.pungi_conf(arch, variant)
@@ -153,9 +154,7 @@ def resolve_deps(compose, arch, variant):
         rmtree(tmp_dir)
 
     with open(pungi_log, "r") as f:
-        result = pungi_wrapper.get_packages(f)
+        packages, broken_deps, missing_comps_pkgs = pungi_wrapper.parse_log(f)
-        f.seek(0)
-        missing_comps_pkgs = pungi_wrapper.get_missing_comps_packages(f)
 
     if missing_comps_pkgs:
         msg = ("Packages mentioned in comps do not exist for %s.%s: %s"
@@ -165,17 +164,13 @@ def resolve_deps(compose, arch, variant):
             raise RuntimeError(msg)
 
     compose.log_info("[DONE ] %s" % msg)
-    return result
+    return packages, broken_deps
 
 
-def check_deps(compose, arch, variant):
+def check_deps(compose, arch, variant, missing_deps):
     if not compose.conf["check_deps"]:
         return
 
-    pungi_wrapper = PungiWrapper()
-    pungi_log = compose.paths.work.pungi_log(arch, variant)
-    with open(pungi_log, "r") as f:
-        missing_deps = pungi_wrapper.get_missing_deps(f)
     if missing_deps:
         for pkg in sorted(missing_deps):
             compose.log_error("Unresolved dependencies in package %s: %s" % (pkg, sorted(missing_deps[pkg])))

pungi/wrappers/pungi.py

@@ -202,8 +202,10 @@ class PungiWrapper(object):
 
         return cmd
 
-    def get_packages(self, f):
+    def parse_log(self, f):
-        result = dict(((i, []) for i in PACKAGES_RE))
+        packages = dict(((i, []) for i in PACKAGES_RE))
+        broken_deps = {}
+        missing_comps = set()
 
         for line in f:
             for file_type, pattern in PACKAGES_RE.iteritems():
@@ -216,30 +218,18 @@ class PungiWrapper(object):
                     flags = match.groupdict()["flags"] or ""
                     flags = sorted([i.strip() for i in flags.split(",") if i.strip()])
                     item["flags"] = flags
-                    result[file_type].append(item)
+                    packages[file_type].append(item)
                     break
 
-        # no packages are filtered
-
-        return result
-
-    def get_missing_comps_packages(self, f):
-        result = set()
-        for line in f:
             match = MISSING_COMPS_PACKAGE_RE.match(line)
             if match:
-                result.add(match.group(1))
+                missing_comps.add(match.group(1))
-        return result
 
-    def get_missing_deps(self, f):
-        result = {}
-
-        for line in f:
             match = UNRESOLVED_DEPENDENCY_RE.match(line)
             if match:
-                result.setdefault(match.group(2), set()).add(match.group(1))
+                broken_deps.setdefault(match.group(2), set()).add(match.group(1))
 
-        return result
+        return packages, broken_deps, missing_comps
 
     def run_pungi(self, ks_file, destdir, name, selfhosting=False, fulltree=False,
                   greedy='', cache_dir=None, arch='', multilib_methods=[],

tests/test_gather.py

@@ -1618,7 +1618,7 @@ class PungiYumDepsolvingTestCase(DepsolvingBase, unittest.TestCase):
 
         p.run_pungi(self.ks, self.tmp_dir, 'DP', **kwargs)
         with open(self.out, "r") as f:
-            pkg_map = p.get_packages(f)
+            pkg_map, _, _ = p.parse_log(f)
         return convert_pkg_map(pkg_map)