almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

Integrate DNF gathering code with Pungi.

Browse Source 
Set 'dnf_gather = True' (temporary option) in config file to enable DNF
gathering.

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Daniel Mach 2015-07-10 06:43:06 -04:00 committed by Lubomír Sedlář
parent 9764acbc4d
commit 43d8d02cb1
5 changed files with 80 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
bin/pungi-gather
View File

@ -69,15 +69,6 @@ def main():
dnf_conf = Conf(ns.arch)
dnf_obj = DnfWrapper(dnf_conf)
ksparser = pungi.ks.get_ksparser(ns.config)
# read repos from ks
for ks_repo in ksparser.handler.repo.repoList:
dnf_obj.add_repo(ks_repo.name, ks_repo.baseurl)
dnf_obj.fill_sack(load_system_repo=False, load_available_repos=True)
dnf_obj.read_comps()
gather_opts = GatherOptions()
if ns.greedy:
@ -89,6 +80,23 @@ def main():
if ns.lookaside:
gather_opts.lookaside_repos = ns.lookaside
ksparser = pungi.ks.get_ksparser(ns.config)
# read repos from ks
for ks_repo in ksparser.handler.repo.repoList:
# HACK: lookaside repos first; this is workaround for no repo priority handling in hawkey
if ks_repo.name not in gather_opts.lookaside_repos:
continue
dnf_obj.add_repo(ks_repo.name, ks_repo.baseurl)
for ks_repo in ksparser.handler.repo.repoList:
if ks_repo.name in gather_opts.lookaside_repos:
continue
dnf_obj.add_repo(ks_repo.name, ks_repo.baseurl)
dnf_obj.fill_sack(load_system_repo=False, load_available_repos=True)
dnf_obj.read_comps()
gather_opts.multilib_blacklist = ksparser.handler.multilib_blacklist
gather_opts.multilib_whitelist = ksparser.handler.multilib_whitelist
gather_opts.prepopulate = ksparser.handler.prepopulate
@ -113,15 +121,23 @@ def _get_flags(gather_obj, pkg):
return flags
def _get_url(pkg):
if pkg.baseurl:
result = os.path.join(pkg.baseurl, pkg.location)
else:
result = os.path.join(pkg.repo.baseurl[0], pkg.location)
return result
def print_rpms(gather_obj):
for pkg in sorted(gather_obj.result_binary_packages):
print "RPM%s: %s-%s-%s.%s" % (_get_flags(gather_obj, pkg), pkg.name, pkg.version, pkg.release, pkg.arch)
print "RPM%s: %s" % (_get_flags(gather_obj, pkg), _get_url(pkg))
for pkg in sorted(gather_obj.result_debug_packages):
print "DEBUGINFO%s: %s-%s-%s.%s" % (_get_flags(gather_obj, pkg), pkg.name, pkg.version, pkg.release, pkg.arch)
print "DEBUGINFO%s: %s" % (_get_flags(gather_obj, pkg), _get_url(pkg))
for pkg in sorted(gather_obj.result_source_packages):
print "SRPM%s: %s-%s-%s.%s" % (_get_flags(gather_obj, pkg), pkg.name, pkg.version, pkg.release, pkg.arch)
print "SRPM%s: %s" % (_get_flags(gather_obj, pkg), _get_url(pkg))
if __name__ == "__main__":

2
pungi/gather_dnf.py
View File

@ -267,7 +267,7 @@ class Gather(GatherBase):
added.add(i)
pb = ""
if pulled_by:
pb = " (pulled by %s)" % pulled_by
pb = " (pulled by %s, repo: %s)" % (pulled_by, pulled_by.repo.id)
print "Added package %s%s" % (i, pb)
self.result_binary_packages.add(i)
# lookaside

6
pungi/phases/gather/link.py
View File

@ -70,6 +70,8 @@ def link_files(compose, arch, variant, pkg_map, pkg_sets, manifest, srpm_map={})
packages_dir = compose.paths.compose.packages("src", variant)
packages_dir_relpath = compose.paths.compose.packages("src", variant, relative=True)
for pkg in pkg_map["srpm"]:
if "lookaside" in pkg["flags"]:
continue
dst = os.path.join(packages_dir, get_package_path(os.path.basename(pkg["path"]), hashed_directories))
dst_relpath = os.path.join(packages_dir_relpath, get_package_path(os.path.basename(pkg["path"]), hashed_directories))
@ -87,6 +89,8 @@ def link_files(compose, arch, variant, pkg_map, pkg_sets, manifest, srpm_map={})
packages_dir = compose.paths.compose.packages(arch, variant)
packages_dir_relpath = compose.paths.compose.packages(arch, variant, relative=True)
for pkg in pkg_map["rpm"]:
if "lookaside" in pkg["flags"]:
continue
dst = os.path.join(packages_dir, get_package_path(os.path.basename(pkg["path"]), hashed_directories))
dst_relpath = os.path.join(packages_dir_relpath, get_package_path(os.path.basename(pkg["path"]), hashed_directories))
@ -102,6 +106,8 @@ def link_files(compose, arch, variant, pkg_map, pkg_sets, manifest, srpm_map={})
packages_dir = compose.paths.compose.debug_packages(arch, variant)
packages_dir_relpath = compose.paths.compose.debug_packages(arch, variant, relative=True)
for pkg in pkg_map["debuginfo"]:
if "lookaside" in pkg["flags"]:
continue
dst = os.path.join(packages_dir, get_package_path(os.path.basename(pkg["path"]), hashed_directories))
dst_relpath = os.path.join(packages_dir_relpath, get_package_path(os.path.basename(pkg["path"]), hashed_directories))

6
pungi/phases/gather/methods/method_deps.py
View File

@ -118,7 +118,11 @@ def resolve_deps(compose, arch, variant):
yum_arch = tree_arch_to_yum_arch(arch)
tmp_dir = compose.paths.work.tmp_dir(arch, variant)
cache_dir = compose.paths.work.pungi_cache_dir(arch, variant)
cmd = pungi_wrapper.get_pungi_cmd(pungi_conf, destdir=tmp_dir, name=variant.uid, selfhosting=selfhosting, fulltree=fulltree, arch=yum_arch, full_archlist=True, greedy=greedy_method, cache_dir=cache_dir, lookaside_repos=lookaside_repos, multilib_methods=multilib_methods)
# TODO: remove YUM code, fully migrate to DNF
if compose.conf.get("dnf_gather", False):
cmd = pungi_wrapper.get_pungi_cmd_dnf(pungi_conf, destdir=tmp_dir, name=variant.uid, selfhosting=selfhosting, fulltree=fulltree, arch=yum_arch, full_archlist=True, greedy=greedy_method, cache_dir=cache_dir, lookaside_repos=lookaside_repos, multilib_methods=multilib_methods)
else:
cmd = pungi_wrapper.get_pungi_cmd(pungi_conf, destdir=tmp_dir, name=variant.uid, selfhosting=selfhosting, fulltree=fulltree, arch=yum_arch, full_archlist=True, greedy=greedy_method, cache_dir=cache_dir, lookaside_repos=lookaside_repos, multilib_methods=multilib_methods)
# Use temp working directory directory as workaround for
# https://bugzilla.redhat.com/show_bug.cgi?id=795137
tmp_dir = compose.mkdtemp(prefix="pungi_")

44
pungi/wrappers/pungi.py
View File

@ -21,9 +21,9 @@ from .. import util
PACKAGES_RE = {
"rpm": re.compile(r"^RPM(\((?P<flags>[^\)]+)\))?: (?:file://)?(?P<path>/?[^ ]+)$"),
"srpm": re.compile(r"^SRPM(\((?P<flags>[^\)]+)\))?: (?:file://)?(?P<path>/?[^ ]+)$"),
"debuginfo": re.compile(r"^DEBUGINFO(\((?P<flags>[^\)]+)\))?: (?:file://)?(?P<path>/?[^ ]+)$"),
"rpm": re.compile(r"^RPM(\((?P<flags>[^\)]*)\))?: (?P<path>.+)$"),
"srpm": re.compile(r"^SRPM(\((?P<flags>[^\)]*)\))?: (?P<path>.+)$"),
"debuginfo": re.compile(r"^DEBUGINFO(\((?P<flags>[^\)]*)\))?: (?P<path>.+)$"),
}
@ -167,6 +167,40 @@ class PungiWrapper(object):
return cmd
def get_pungi_cmd_dnf(self, config, destdir, name, version=None, flavor=None, selfhosting=False, fulltree=False, greedy=None, nodeps=False, nodownload=True, full_archlist=False, arch=None, cache_dir=None, lookaside_repos=None, multilib_methods=None):
cmd = ["pungi-gather"]
# path to a kickstart file
cmd.append("--config=%s" % config)
# turn selfhosting on
if selfhosting:
cmd.append("--selfhosting")
# NPLB
if fulltree:
cmd.append("--fulltree")
greedy = greedy or "none"
cmd.append("--greedy=%s" % greedy)
if nodeps:
cmd.append("--nodeps")
if arch:
cmd.append("--arch=%s" % arch)
if multilib_methods:
for i in multilib_methods:
cmd.append("--multilib=%s" % i)
if lookaside_repos:
for i in lookaside_repos:
cmd.append("--lookaside=%s" % i)
return cmd
def get_packages(self, output):
global PACKAGES_RE
result = dict(((i, []) for i in PACKAGES_RE))
@ -176,7 +210,9 @@ class PungiWrapper(object):
match = pattern.match(line)
if match:
item = {}
item["path"] = match.groupdict()["path"]
item["path"] = match.groupdict()["path"].strip()
if item["path"].startswith("file://"):
item["path"] = item["path"][7:]
flags = match.groupdict()["flags"] or ""
flags = sorted([i.strip() for i in flags.split(",") if i.strip()])
item["flags"] = flags