gather: Avoid reading whole log into memory

There's no reason for reading the whole log of depsolving into memory
just to split it into lines and process one line at a time.

We can just as well read it in chunks.

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

contrib/yum-dnf-compare/pungi-compare-depsolving

@@ -18,7 +18,8 @@ from pungi.wrappers.pungi import PungiWrapper
 
 def read_rpms(fn):
     pw = PungiWrapper()
-    data = pw.get_packages(open(fn, "r").read())
+    with open(fn, "r") as f:
+        data = pw.get_packages(f)
     result = set()
     for i in data["rpm"]:
         nvra = parse_nvra(i["path"])

pungi/phases/gather/methods/method_deps.py

@@ -102,7 +102,8 @@ def resolve_deps(compose, arch, variant):
     msg = "Running pungi (arch: %s, variant: %s)" % (arch, variant)
     if compose.DEBUG and os.path.exists(pungi_log):
         compose.log_warning("[SKIP ] %s" % msg)
-        return pungi_wrapper.get_packages(open(pungi_log, "r").read())
+        with open(pungi_log, "r") as f:
+            return pungi_wrapper.get_packages(f)
 
     compose.log_info("[BEGIN] %s" % msg)
     pungi_conf = compose.paths.work.pungi_conf(arch, variant)
@@ -150,7 +151,9 @@ def resolve_deps(compose, arch, variant):
         run(cmd, logfile=pungi_log, show_cmd=True, workdir=tmp_dir, env=os.environ)
     finally:
         rmtree(tmp_dir)
-    result = pungi_wrapper.get_packages(open(pungi_log, "r").read())
+
+    with open(pungi_log, "r") as f:
+        result = pungi_wrapper.get_packages(f)
 
     compose.log_info("[DONE ] %s" % msg)
     return result
@@ -162,7 +165,8 @@ def check_deps(compose, arch, variant):
 
     pungi_wrapper = PungiWrapper()
     pungi_log = compose.paths.work.pungi_log(arch, variant)
-    missing_deps = pungi_wrapper.get_missing_deps(open(pungi_log, "r").read())
+    with open(pungi_log, "r") as f:
+        missing_deps = pungi_wrapper.get_missing_deps(f)
     if missing_deps:
         for pkg in sorted(missing_deps):
             compose.log_error("Unresolved dependencies in package %s: %s" % (pkg, sorted(missing_deps[pkg])))

pungi/wrappers/pungi.py

@@ -200,11 +200,10 @@ class PungiWrapper(object):
 
         return cmd
 
-    def get_packages(self, output):
-        global PACKAGES_RE
+    def get_packages(self, f):
         result = dict(((i, []) for i in PACKAGES_RE))
 
-        for line in output.splitlines():
+        for line in f:
             for file_type, pattern in PACKAGES_RE.iteritems():
                 match = pattern.match(line)
                 if match:
@@ -222,11 +221,10 @@ class PungiWrapper(object):
 
         return result
 
-    def get_missing_deps(self, output):
-        global UNRESOLVED_DEPENDENCY_RE
+    def get_missing_deps(self, f):
         result = {}
 
-        for line in output.splitlines():
+        for line in f:
             match = UNRESOLVED_DEPENDENCY_RE.match(line)
             if match:
                 result.setdefault(match.group(2), set()).add(match.group(1))

tests/test_gather.py

@@ -1618,7 +1618,7 @@ class PungiYumDepsolvingTestCase(DepsolvingBase, unittest.TestCase):
 
         p.run_pungi(self.ks, self.tmp_dir, 'DP', **kwargs)
         with open(self.out, "r") as f:
-            pkg_map = p.get_packages(f.read())
+            pkg_map = p.get_packages(f)
         return convert_pkg_map(pkg_map)