diff --git a/bin/pungi-gather b/bin/pungi-gather index 6cbf1a57..c8fce473 100755 --- a/bin/pungi-gather +++ b/bin/pungi-gather @@ -69,15 +69,6 @@ def main(): dnf_conf = Conf(ns.arch) dnf_obj = DnfWrapper(dnf_conf) - ksparser = pungi.ks.get_ksparser(ns.config) - - # read repos from ks - for ks_repo in ksparser.handler.repo.repoList: - dnf_obj.add_repo(ks_repo.name, ks_repo.baseurl) - - dnf_obj.fill_sack(load_system_repo=False, load_available_repos=True) - dnf_obj.read_comps() - gather_opts = GatherOptions() if ns.greedy: @@ -89,6 +80,23 @@ def main(): if ns.lookaside: gather_opts.lookaside_repos = ns.lookaside + ksparser = pungi.ks.get_ksparser(ns.config) + + # read repos from ks + for ks_repo in ksparser.handler.repo.repoList: + # HACK: lookaside repos first; this is workaround for no repo priority handling in hawkey + if ks_repo.name not in gather_opts.lookaside_repos: + continue + dnf_obj.add_repo(ks_repo.name, ks_repo.baseurl) + + for ks_repo in ksparser.handler.repo.repoList: + if ks_repo.name in gather_opts.lookaside_repos: + continue + dnf_obj.add_repo(ks_repo.name, ks_repo.baseurl) + + dnf_obj.fill_sack(load_system_repo=False, load_available_repos=True) + dnf_obj.read_comps() + gather_opts.multilib_blacklist = ksparser.handler.multilib_blacklist gather_opts.multilib_whitelist = ksparser.handler.multilib_whitelist gather_opts.prepopulate = ksparser.handler.prepopulate @@ -113,15 +121,23 @@ def _get_flags(gather_obj, pkg): return flags +def _get_url(pkg): + if pkg.baseurl: + result = os.path.join(pkg.baseurl, pkg.location) + else: + result = os.path.join(pkg.repo.baseurl[0], pkg.location) + return result + + def print_rpms(gather_obj): for pkg in sorted(gather_obj.result_binary_packages): - print "RPM%s: %s-%s-%s.%s" % (_get_flags(gather_obj, pkg), pkg.name, pkg.version, pkg.release, pkg.arch) + print "RPM%s: %s" % (_get_flags(gather_obj, pkg), _get_url(pkg)) for pkg in sorted(gather_obj.result_debug_packages): - print "DEBUGINFO%s: %s-%s-%s.%s" % (_get_flags(gather_obj, pkg), pkg.name, pkg.version, pkg.release, pkg.arch) + print "DEBUGINFO%s: %s" % (_get_flags(gather_obj, pkg), _get_url(pkg)) for pkg in sorted(gather_obj.result_source_packages): - print "SRPM%s: %s-%s-%s.%s" % (_get_flags(gather_obj, pkg), pkg.name, pkg.version, pkg.release, pkg.arch) + print "SRPM%s: %s" % (_get_flags(gather_obj, pkg), _get_url(pkg)) if __name__ == "__main__": diff --git a/pungi/gather_dnf.py b/pungi/gather_dnf.py index 626e7a3f..f57ffce0 100644 --- a/pungi/gather_dnf.py +++ b/pungi/gather_dnf.py @@ -267,7 +267,7 @@ class Gather(GatherBase): added.add(i) pb = "" if pulled_by: - pb = " (pulled by %s)" % pulled_by + pb = " (pulled by %s, repo: %s)" % (pulled_by, pulled_by.repo.id) print "Added package %s%s" % (i, pb) self.result_binary_packages.add(i) # lookaside diff --git a/pungi/phases/gather/link.py b/pungi/phases/gather/link.py index 82f00fcc..29921654 100644 --- a/pungi/phases/gather/link.py +++ b/pungi/phases/gather/link.py @@ -70,6 +70,8 @@ def link_files(compose, arch, variant, pkg_map, pkg_sets, manifest, srpm_map={}) packages_dir = compose.paths.compose.packages("src", variant) packages_dir_relpath = compose.paths.compose.packages("src", variant, relative=True) for pkg in pkg_map["srpm"]: + if "lookaside" in pkg["flags"]: + continue dst = os.path.join(packages_dir, get_package_path(os.path.basename(pkg["path"]), hashed_directories)) dst_relpath = os.path.join(packages_dir_relpath, get_package_path(os.path.basename(pkg["path"]), hashed_directories)) @@ -87,6 +89,8 @@ def link_files(compose, arch, variant, pkg_map, pkg_sets, manifest, srpm_map={}) packages_dir = compose.paths.compose.packages(arch, variant) packages_dir_relpath = compose.paths.compose.packages(arch, variant, relative=True) for pkg in pkg_map["rpm"]: + if "lookaside" in pkg["flags"]: + continue dst = os.path.join(packages_dir, get_package_path(os.path.basename(pkg["path"]), hashed_directories)) dst_relpath = os.path.join(packages_dir_relpath, get_package_path(os.path.basename(pkg["path"]), hashed_directories)) @@ -102,6 +106,8 @@ def link_files(compose, arch, variant, pkg_map, pkg_sets, manifest, srpm_map={}) packages_dir = compose.paths.compose.debug_packages(arch, variant) packages_dir_relpath = compose.paths.compose.debug_packages(arch, variant, relative=True) for pkg in pkg_map["debuginfo"]: + if "lookaside" in pkg["flags"]: + continue dst = os.path.join(packages_dir, get_package_path(os.path.basename(pkg["path"]), hashed_directories)) dst_relpath = os.path.join(packages_dir_relpath, get_package_path(os.path.basename(pkg["path"]), hashed_directories)) diff --git a/pungi/phases/gather/methods/method_deps.py b/pungi/phases/gather/methods/method_deps.py index b2e8f9a8..e1249daf 100644 --- a/pungi/phases/gather/methods/method_deps.py +++ b/pungi/phases/gather/methods/method_deps.py @@ -118,7 +118,11 @@ def resolve_deps(compose, arch, variant): yum_arch = tree_arch_to_yum_arch(arch) tmp_dir = compose.paths.work.tmp_dir(arch, variant) cache_dir = compose.paths.work.pungi_cache_dir(arch, variant) - cmd = pungi_wrapper.get_pungi_cmd(pungi_conf, destdir=tmp_dir, name=variant.uid, selfhosting=selfhosting, fulltree=fulltree, arch=yum_arch, full_archlist=True, greedy=greedy_method, cache_dir=cache_dir, lookaside_repos=lookaside_repos, multilib_methods=multilib_methods) + # TODO: remove YUM code, fully migrate to DNF + if compose.conf.get("dnf_gather", False): + cmd = pungi_wrapper.get_pungi_cmd_dnf(pungi_conf, destdir=tmp_dir, name=variant.uid, selfhosting=selfhosting, fulltree=fulltree, arch=yum_arch, full_archlist=True, greedy=greedy_method, cache_dir=cache_dir, lookaside_repos=lookaside_repos, multilib_methods=multilib_methods) + else: + cmd = pungi_wrapper.get_pungi_cmd(pungi_conf, destdir=tmp_dir, name=variant.uid, selfhosting=selfhosting, fulltree=fulltree, arch=yum_arch, full_archlist=True, greedy=greedy_method, cache_dir=cache_dir, lookaside_repos=lookaside_repos, multilib_methods=multilib_methods) # Use temp working directory directory as workaround for # https://bugzilla.redhat.com/show_bug.cgi?id=795137 tmp_dir = compose.mkdtemp(prefix="pungi_") diff --git a/pungi/wrappers/pungi.py b/pungi/wrappers/pungi.py index 32033a6a..21635c99 100644 --- a/pungi/wrappers/pungi.py +++ b/pungi/wrappers/pungi.py @@ -21,9 +21,9 @@ from .. import util PACKAGES_RE = { - "rpm": re.compile(r"^RPM(\((?P[^\)]+)\))?: (?:file://)?(?P/?[^ ]+)$"), - "srpm": re.compile(r"^SRPM(\((?P[^\)]+)\))?: (?:file://)?(?P/?[^ ]+)$"), - "debuginfo": re.compile(r"^DEBUGINFO(\((?P[^\)]+)\))?: (?:file://)?(?P/?[^ ]+)$"), + "rpm": re.compile(r"^RPM(\((?P[^\)]*)\))?: (?P.+)$"), + "srpm": re.compile(r"^SRPM(\((?P[^\)]*)\))?: (?P.+)$"), + "debuginfo": re.compile(r"^DEBUGINFO(\((?P[^\)]*)\))?: (?P.+)$"), } @@ -167,6 +167,40 @@ class PungiWrapper(object): return cmd + def get_pungi_cmd_dnf(self, config, destdir, name, version=None, flavor=None, selfhosting=False, fulltree=False, greedy=None, nodeps=False, nodownload=True, full_archlist=False, arch=None, cache_dir=None, lookaside_repos=None, multilib_methods=None): + cmd = ["pungi-gather"] + + # path to a kickstart file + cmd.append("--config=%s" % config) + + # turn selfhosting on + if selfhosting: + cmd.append("--selfhosting") + + # NPLB + if fulltree: + cmd.append("--fulltree") + + greedy = greedy or "none" + cmd.append("--greedy=%s" % greedy) + + if nodeps: + cmd.append("--nodeps") + + if arch: + cmd.append("--arch=%s" % arch) + + if multilib_methods: + for i in multilib_methods: + cmd.append("--multilib=%s" % i) + + if lookaside_repos: + for i in lookaside_repos: + cmd.append("--lookaside=%s" % i) + + return cmd + + def get_packages(self, output): global PACKAGES_RE result = dict(((i, []) for i in PACKAGES_RE)) @@ -176,7 +210,9 @@ class PungiWrapper(object): match = pattern.match(line) if match: item = {} - item["path"] = match.groupdict()["path"] + item["path"] = match.groupdict()["path"].strip() + if item["path"].startswith("file://"): + item["path"] = item["path"][7:] flags = match.groupdict()["flags"] or "" flags = sorted([i.strip() for i in flags.split(",") if i.strip()]) item["flags"] = flags