gather: Only parse pungi log once

No need to read the file three times, we can just get all the data in
one go.

Signed-off-by: Lubomír Sedlář <>
Lubomír Sedlář 6 years ago
parent cef8650c3e
commit c9f34b6684

@ -19,7 +19,7 @@ from pungi.wrappers.pungi import PungiWrapper
def read_rpms(fn):
pw = PungiWrapper()
with open(fn, "r") as f:
data = pw.get_packages(f)
data, _, _ = pw.parse_log(f)
result = set()
for i in data["rpm"]:
nvra = parse_nvra(i["path"])

@ -41,8 +41,8 @@ class GatherMethodDeps(pungi.phases.gather.method.GatherMethodBase):
write_pungi_config(self.compose, arch, variant, packages, groups, filter_packages,
multilib_whitelist, multilib_blacklist,
fulltree_excludes=fulltree_excludes, prepopulate=prepopulate)
result = resolve_deps(self.compose, arch, variant)
check_deps(self.compose, arch, variant)
result, missing_deps = resolve_deps(self.compose, arch, variant)
check_deps(self.compose, arch, variant, missing_deps)
return result
@ -103,7 +103,8 @@ def resolve_deps(compose, arch, variant):
if compose.DEBUG and os.path.exists(pungi_log):
compose.log_warning("[SKIP ] %s" % msg)
with open(pungi_log, "r") as f:
return pungi_wrapper.get_packages(f)
res, broken_deps, _ = pungi_wrapper.parse_log(f)
return res, broken_deps
compose.log_info("[BEGIN] %s" % msg)
pungi_conf =, variant)
@ -153,9 +154,7 @@ def resolve_deps(compose, arch, variant):
with open(pungi_log, "r") as f:
result = pungi_wrapper.get_packages(f)
missing_comps_pkgs = pungi_wrapper.get_missing_comps_packages(f)
packages, broken_deps, missing_comps_pkgs = pungi_wrapper.parse_log(f)
if missing_comps_pkgs:
msg = ("Packages mentioned in comps do not exist for %s.%s: %s"
@ -165,17 +164,13 @@ def resolve_deps(compose, arch, variant):
raise RuntimeError(msg)
compose.log_info("[DONE ] %s" % msg)
return result
return packages, broken_deps
def check_deps(compose, arch, variant):
def check_deps(compose, arch, variant, missing_deps):
if not compose.conf["check_deps"]:
pungi_wrapper = PungiWrapper()
pungi_log =, variant)
with open(pungi_log, "r") as f:
missing_deps = pungi_wrapper.get_missing_deps(f)
if missing_deps:
for pkg in sorted(missing_deps):
compose.log_error("Unresolved dependencies in package %s: %s" % (pkg, sorted(missing_deps[pkg])))

@ -202,8 +202,10 @@ class PungiWrapper(object):
return cmd
def get_packages(self, f):
result = dict(((i, []) for i in PACKAGES_RE))
def parse_log(self, f):
packages = dict(((i, []) for i in PACKAGES_RE))
broken_deps = {}
missing_comps = set()
for line in f:
for file_type, pattern in PACKAGES_RE.iteritems():
@ -216,30 +218,18 @@ class PungiWrapper(object):
flags = match.groupdict()["flags"] or ""
flags = sorted([i.strip() for i in flags.split(",") if i.strip()])
item["flags"] = flags
# no packages are filtered
return result
def get_missing_comps_packages(self, f):
result = set()
for line in f:
match = MISSING_COMPS_PACKAGE_RE.match(line)
if match:
return result
def get_missing_deps(self, f):
result = {}
for line in f:
match = UNRESOLVED_DEPENDENCY_RE.match(line)
if match:
result.setdefault(, set()).add(
broken_deps.setdefault(, set()).add(
return result
return packages, broken_deps, missing_comps
def run_pungi(self, ks_file, destdir, name, selfhosting=False, fulltree=False,
greedy='', cache_dir=None, arch='', multilib_methods=[],

@ -1618,7 +1618,7 @@ class PungiYumDepsolvingTestCase(DepsolvingBase, unittest.TestCase):
p.run_pungi(self.ks, self.tmp_dir, 'DP', **kwargs)
with open(self.out, "r") as f:
pkg_map = p.get_packages(f)
pkg_map, _, _ = p.parse_log(f)
return convert_pkg_map(pkg_map)