gather: Only parse pungi log once

No need to read the file three times, we can just get all the data in
one go.

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2017-08-03 13:42:04 +02:00
parent cef8650c3e
commit c9f34b6684
4 changed files with 17 additions and 32 deletions

View File

@ -19,7 +19,7 @@ from pungi.wrappers.pungi import PungiWrapper
def read_rpms(fn): def read_rpms(fn):
pw = PungiWrapper() pw = PungiWrapper()
with open(fn, "r") as f: with open(fn, "r") as f:
data = pw.get_packages(f) data, _, _ = pw.parse_log(f)
result = set() result = set()
for i in data["rpm"]: for i in data["rpm"]:
nvra = parse_nvra(i["path"]) nvra = parse_nvra(i["path"])

View File

@ -41,8 +41,8 @@ class GatherMethodDeps(pungi.phases.gather.method.GatherMethodBase):
write_pungi_config(self.compose, arch, variant, packages, groups, filter_packages, write_pungi_config(self.compose, arch, variant, packages, groups, filter_packages,
multilib_whitelist, multilib_blacklist, multilib_whitelist, multilib_blacklist,
fulltree_excludes=fulltree_excludes, prepopulate=prepopulate) fulltree_excludes=fulltree_excludes, prepopulate=prepopulate)
result = resolve_deps(self.compose, arch, variant) result, missing_deps = resolve_deps(self.compose, arch, variant)
check_deps(self.compose, arch, variant) check_deps(self.compose, arch, variant, missing_deps)
return result return result
@ -103,7 +103,8 @@ def resolve_deps(compose, arch, variant):
if compose.DEBUG and os.path.exists(pungi_log): if compose.DEBUG and os.path.exists(pungi_log):
compose.log_warning("[SKIP ] %s" % msg) compose.log_warning("[SKIP ] %s" % msg)
with open(pungi_log, "r") as f: with open(pungi_log, "r") as f:
return pungi_wrapper.get_packages(f) res, broken_deps, _ = pungi_wrapper.parse_log(f)
return res, broken_deps
compose.log_info("[BEGIN] %s" % msg) compose.log_info("[BEGIN] %s" % msg)
pungi_conf = compose.paths.work.pungi_conf(arch, variant) pungi_conf = compose.paths.work.pungi_conf(arch, variant)
@ -153,9 +154,7 @@ def resolve_deps(compose, arch, variant):
rmtree(tmp_dir) rmtree(tmp_dir)
with open(pungi_log, "r") as f: with open(pungi_log, "r") as f:
result = pungi_wrapper.get_packages(f) packages, broken_deps, missing_comps_pkgs = pungi_wrapper.parse_log(f)
f.seek(0)
missing_comps_pkgs = pungi_wrapper.get_missing_comps_packages(f)
if missing_comps_pkgs: if missing_comps_pkgs:
msg = ("Packages mentioned in comps do not exist for %s.%s: %s" msg = ("Packages mentioned in comps do not exist for %s.%s: %s"
@ -165,17 +164,13 @@ def resolve_deps(compose, arch, variant):
raise RuntimeError(msg) raise RuntimeError(msg)
compose.log_info("[DONE ] %s" % msg) compose.log_info("[DONE ] %s" % msg)
return result return packages, broken_deps
def check_deps(compose, arch, variant): def check_deps(compose, arch, variant, missing_deps):
if not compose.conf["check_deps"]: if not compose.conf["check_deps"]:
return return
pungi_wrapper = PungiWrapper()
pungi_log = compose.paths.work.pungi_log(arch, variant)
with open(pungi_log, "r") as f:
missing_deps = pungi_wrapper.get_missing_deps(f)
if missing_deps: if missing_deps:
for pkg in sorted(missing_deps): for pkg in sorted(missing_deps):
compose.log_error("Unresolved dependencies in package %s: %s" % (pkg, sorted(missing_deps[pkg]))) compose.log_error("Unresolved dependencies in package %s: %s" % (pkg, sorted(missing_deps[pkg])))

View File

@ -202,8 +202,10 @@ class PungiWrapper(object):
return cmd return cmd
def get_packages(self, f): def parse_log(self, f):
result = dict(((i, []) for i in PACKAGES_RE)) packages = dict(((i, []) for i in PACKAGES_RE))
broken_deps = {}
missing_comps = set()
for line in f: for line in f:
for file_type, pattern in PACKAGES_RE.iteritems(): for file_type, pattern in PACKAGES_RE.iteritems():
@ -216,30 +218,18 @@ class PungiWrapper(object):
flags = match.groupdict()["flags"] or "" flags = match.groupdict()["flags"] or ""
flags = sorted([i.strip() for i in flags.split(",") if i.strip()]) flags = sorted([i.strip() for i in flags.split(",") if i.strip()])
item["flags"] = flags item["flags"] = flags
result[file_type].append(item) packages[file_type].append(item)
break break
# no packages are filtered
return result
def get_missing_comps_packages(self, f):
result = set()
for line in f:
match = MISSING_COMPS_PACKAGE_RE.match(line) match = MISSING_COMPS_PACKAGE_RE.match(line)
if match: if match:
result.add(match.group(1)) missing_comps.add(match.group(1))
return result
def get_missing_deps(self, f):
result = {}
for line in f:
match = UNRESOLVED_DEPENDENCY_RE.match(line) match = UNRESOLVED_DEPENDENCY_RE.match(line)
if match: if match:
result.setdefault(match.group(2), set()).add(match.group(1)) broken_deps.setdefault(match.group(2), set()).add(match.group(1))
return result return packages, broken_deps, missing_comps
def run_pungi(self, ks_file, destdir, name, selfhosting=False, fulltree=False, def run_pungi(self, ks_file, destdir, name, selfhosting=False, fulltree=False,
greedy='', cache_dir=None, arch='', multilib_methods=[], greedy='', cache_dir=None, arch='', multilib_methods=[],

View File

@ -1618,7 +1618,7 @@ class PungiYumDepsolvingTestCase(DepsolvingBase, unittest.TestCase):
p.run_pungi(self.ks, self.tmp_dir, 'DP', **kwargs) p.run_pungi(self.ks, self.tmp_dir, 'DP', **kwargs)
with open(self.out, "r") as f: with open(self.out, "r") as f:
pkg_map = p.get_packages(f) pkg_map, _, _ = p.parse_log(f)
return convert_pkg_map(pkg_map) return convert_pkg_map(pkg_map)