From c9f34b668432aec3d4614b7b274a14b5f25348e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Thu, 3 Aug 2017 13:42:04 +0200 Subject: [PATCH] gather: Only parse pungi log once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No need to read the file three times, we can just get all the data in one go. Signed-off-by: Lubomír Sedlář --- .../yum-dnf-compare/pungi-compare-depsolving | 2 +- pungi/phases/gather/methods/method_deps.py | 19 +++++--------- pungi/wrappers/pungi.py | 26 ++++++------------- tests/test_gather.py | 2 +- 4 files changed, 17 insertions(+), 32 deletions(-) diff --git a/contrib/yum-dnf-compare/pungi-compare-depsolving b/contrib/yum-dnf-compare/pungi-compare-depsolving index eeec2471..4e63beb5 100755 --- a/contrib/yum-dnf-compare/pungi-compare-depsolving +++ b/contrib/yum-dnf-compare/pungi-compare-depsolving @@ -19,7 +19,7 @@ from pungi.wrappers.pungi import PungiWrapper def read_rpms(fn): pw = PungiWrapper() with open(fn, "r") as f: - data = pw.get_packages(f) + data, _, _ = pw.parse_log(f) result = set() for i in data["rpm"]: nvra = parse_nvra(i["path"]) diff --git a/pungi/phases/gather/methods/method_deps.py b/pungi/phases/gather/methods/method_deps.py index 99be5d83..432e9e9d 100644 --- a/pungi/phases/gather/methods/method_deps.py +++ b/pungi/phases/gather/methods/method_deps.py @@ -41,8 +41,8 @@ class GatherMethodDeps(pungi.phases.gather.method.GatherMethodBase): write_pungi_config(self.compose, arch, variant, packages, groups, filter_packages, multilib_whitelist, multilib_blacklist, fulltree_excludes=fulltree_excludes, prepopulate=prepopulate) - result = resolve_deps(self.compose, arch, variant) - check_deps(self.compose, arch, variant) + result, missing_deps = resolve_deps(self.compose, arch, variant) + check_deps(self.compose, arch, variant, missing_deps) return result @@ -103,7 +103,8 @@ def resolve_deps(compose, arch, variant): if compose.DEBUG and os.path.exists(pungi_log): compose.log_warning("[SKIP ] %s" % msg) with open(pungi_log, "r") as f: - return pungi_wrapper.get_packages(f) + res, broken_deps, _ = pungi_wrapper.parse_log(f) + return res, broken_deps compose.log_info("[BEGIN] %s" % msg) pungi_conf = compose.paths.work.pungi_conf(arch, variant) @@ -153,9 +154,7 @@ def resolve_deps(compose, arch, variant): rmtree(tmp_dir) with open(pungi_log, "r") as f: - result = pungi_wrapper.get_packages(f) - f.seek(0) - missing_comps_pkgs = pungi_wrapper.get_missing_comps_packages(f) + packages, broken_deps, missing_comps_pkgs = pungi_wrapper.parse_log(f) if missing_comps_pkgs: msg = ("Packages mentioned in comps do not exist for %s.%s: %s" @@ -165,17 +164,13 @@ def resolve_deps(compose, arch, variant): raise RuntimeError(msg) compose.log_info("[DONE ] %s" % msg) - return result + return packages, broken_deps -def check_deps(compose, arch, variant): +def check_deps(compose, arch, variant, missing_deps): if not compose.conf["check_deps"]: return - pungi_wrapper = PungiWrapper() - pungi_log = compose.paths.work.pungi_log(arch, variant) - with open(pungi_log, "r") as f: - missing_deps = pungi_wrapper.get_missing_deps(f) if missing_deps: for pkg in sorted(missing_deps): compose.log_error("Unresolved dependencies in package %s: %s" % (pkg, sorted(missing_deps[pkg]))) diff --git a/pungi/wrappers/pungi.py b/pungi/wrappers/pungi.py index bbf696ab..1819228b 100644 --- a/pungi/wrappers/pungi.py +++ b/pungi/wrappers/pungi.py @@ -202,8 +202,10 @@ class PungiWrapper(object): return cmd - def get_packages(self, f): - result = dict(((i, []) for i in PACKAGES_RE)) + def parse_log(self, f): + packages = dict(((i, []) for i in PACKAGES_RE)) + broken_deps = {} + missing_comps = set() for line in f: for file_type, pattern in PACKAGES_RE.iteritems(): @@ -216,30 +218,18 @@ class PungiWrapper(object): flags = match.groupdict()["flags"] or "" flags = sorted([i.strip() for i in flags.split(",") if i.strip()]) item["flags"] = flags - result[file_type].append(item) + packages[file_type].append(item) break - # no packages are filtered - - return result - - def get_missing_comps_packages(self, f): - result = set() - for line in f: match = MISSING_COMPS_PACKAGE_RE.match(line) if match: - result.add(match.group(1)) - return result + missing_comps.add(match.group(1)) - def get_missing_deps(self, f): - result = {} - - for line in f: match = UNRESOLVED_DEPENDENCY_RE.match(line) if match: - result.setdefault(match.group(2), set()).add(match.group(1)) + broken_deps.setdefault(match.group(2), set()).add(match.group(1)) - return result + return packages, broken_deps, missing_comps def run_pungi(self, ks_file, destdir, name, selfhosting=False, fulltree=False, greedy='', cache_dir=None, arch='', multilib_methods=[], diff --git a/tests/test_gather.py b/tests/test_gather.py index 15395611..03076f63 100644 --- a/tests/test_gather.py +++ b/tests/test_gather.py @@ -1618,7 +1618,7 @@ class PungiYumDepsolvingTestCase(DepsolvingBase, unittest.TestCase): p.run_pungi(self.ks, self.tmp_dir, 'DP', **kwargs) with open(self.out, "r") as f: - pkg_map = p.get_packages(f) + pkg_map, _, _ = p.parse_log(f) return convert_pkg_map(pkg_map)