almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

Resolve multilib packages. Controlled by the --multilib option.

Browse Source
This commit is contained in:
Daniel Mach 2012-11-26 02:32:01 -05:00 committed by Dennis Gilmore
parent 91f70cbb43
commit 30c0f358d9
12 changed files with 678 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
setup.py
View File

@ -11,6 +11,10 @@ setup(name='pungi',
package_dir = {'': 'src'},
packages = ['pypungi'],
scripts = ['src/bin/pungi.py'],
data_files=[('/usr/share/pungi', glob.glob('share/*'))]
)
data_files=[
('/usr/share/pungi', glob.glob('share/*.xsl')),
('/usr/share/pungi', glob.glob('share/*.ks')),
('/usr/share/pungi/multilib', glob.glob('share/multilib/*')),
]
)

9
share/multilib/devel-blacklist.conf Normal file
View File

@ -0,0 +1,9 @@
dmraid-devel
kdeutils-devel
kernel-devel
mkinitrd-devel
java-1.5.0-gcj-devel
java-1.6.0-openjdk-devel
java-1.7.0-icedtea-devel
java-1.7.0-openjdk-devel
php-devel

2
share/multilib/devel-whitelist.conf Normal file
View File

@ -0,0 +1,2 @@
glibc-static
libstdc++-static

2
share/multilib/runtime-blacklist.conf Normal file
View File

@ -0,0 +1,2 @@
kernel
tomcat-native

78
share/multilib/runtime-patterns.conf Normal file
View File

@ -0,0 +1,78 @@
# format: <path> <filename_wildcard>
# * if filename_wildcard is set to -, then only a directory is matched
# $LIBDIR gets expanded to /lib*, /usr/lib*
# libraries in standard dirs
$LIBDIR *.so.*
$LIBDIR/* *.so.*
# dri
$LIBDIR/dri -
# krb5
$LIBDIR/krb5/plugins -
# pam
$LIBDIR/security -
# sasl
$LIBDIR/sasl2 -
# nss - include nss plugins incl. libnss_*.so
$LIBDIR libnss_*.so
# alsa plugins
$LIBDIR/alsa-lib -
# lsb
/etc/lsb-release.d -
# mysql, qt, etc.
/etc/ld.so.conf.d *.conf
# gtk2-engines
$LIBDIR/gtk-2.0/*/engines -
# accessibility
$LIBDIR/gtk-2.0/modules -
$LIBDIR/gtk-2.0/*/modules -
# scim-bridge-gtk
$LIBDIR/gtk-2.0/immodules -
$LIBDIR/gtk-2.0/*/immodules -
# images
$LIBDIR/gtk-2.0/*/loaders -
$LIBDIR/gdk-pixbuf-2.0/*/loaders -
$LIBDIR/gtk-2.0/*/printbackends -
$LIBDIR/gtk-2.0/*/filesystems -
# qt plugins
$LIBDIR/qt*/plugins/* -
# KDE plugins
$LIBDIR/kde*/plugins/* -
# gstreamer
$LIBDIR/gstreamer-* -
# xine-lib
$LIBDIR/xine/plugins/* -
# oprofile
$LIBDIR/oprofile *.so.*
# wine
$LIBDIR/wine *.so'
# db
$LIBDIR libdb-*
# sane drivers
$LIBDIR/sane libsane-*
# opencryptoki
$LIBDIR/opencryptoki -
# openssl engines
$LIBDIR/openssl/engines *.so

12
share/multilib/runtime-whitelist.conf Normal file
View File

@ -0,0 +1,12 @@
glibc-static
libflashsupport
libgnat
lmms-vst
nspluginwrapper
perl-libs
pulseaudio-utils
redhat-lsb
valgrind
wine
wine-arts
yaboot

4
src/bin/pungi.py
View File

@ -87,6 +87,8 @@ def main():
config.set('pungi', 'full_archlist', "True")
if opts.arch:
config.set('pungi', 'arch', opts.arch)
if opts.multilib:
config.set('pungi', 'multilib', " ".join(opts.multilib))
if opts.lookaside_repos:
config.set('pungi', 'lookaside_repos', " ".join(opts.lookaside_repos))
@ -224,6 +226,8 @@ if __name__ == '__main__':
help='Use the full arch list for x86_64 (include i686, i386, etc.)')
parser.add_option("--arch",
help='Override default (uname based) arch')
parser.add_option("--multilib", action="append", metavar="METHOD",
help='Multilib method; can be specified multiple times; recommended: devel, runtime')
parser.add_option("--lookaside-repo", action="append", dest="lookaside_repos", metavar="NAME",
help='Specify lookaside repo name(s) (packages will used for depsolving but not be included in the output)')

86
src/pypungi/__init__.py
View File

@ -31,6 +31,7 @@ import pylorax
from fnmatch import fnmatch
import arch as arch_module
import multilib
def is_debug(po):
@ -45,6 +46,12 @@ def is_source(po):
return False
def is_noarch(po):
if po.arch == "noarch":
return True
return False
def is_package(po):
if is_debug(po):
return False
@ -184,6 +191,7 @@ class Pungi(pypungi.PungiBase):
self.resolved_deps = {} # list the deps we've already resolved, short circuit.
self.excluded_pkgs = {} # list the packages we've already excluded.
self.seen_pkgs = {} # list the packages we've already seen so we can check all deps only once
self.multilib_methods = self.config.get('pungi', 'multilib').split(" ")
self.lookaside_repos = self.config.get('pungi', 'lookaside_repos').split(" ")
self.sourcerpm_arch_map = {} # {sourcerpm: set[arches]} - used for gathering debuginfo
@ -394,10 +402,10 @@ class Pungi(pypungi.PungiBase):
reqs = po.requires
provs = po.provides
added = []
added = set()
# get langpacks for each processed package
added.extend(self.getLangpacks(po))
added.update(self.getLangpacks([po]))
added.update(self.getMultilib([po]))
for req in reqs:
if req in self.resolved_deps:
@ -424,7 +432,7 @@ class Pungi(pypungi.PungiBase):
if dep not in added:
msg = 'Added %s.%s for %s.%s' % (dep.name, dep.arch, po.name, po.arch)
self.add_package(dep, msg)
added.append(dep)
added.add(dep)
except (yum.Errors.InstallError, yum.Errors.YumBaseError), ex:
self.logger.warn("Unresolvable dependency %s in %s.%s" % (r, po.name, po.arch))
@ -434,34 +442,64 @@ class Pungi(pypungi.PungiBase):
for add in added:
self.getPackageDeps(add)
def getLangpacks(self, po):
def getLangpacks(self, po_list):
added = []
# get all langpacks matching the package name
langpacks = [ i for i in self.langpacks if i["name"] == po.name ]
if not langpacks:
return []
for po in po_list:
# get all langpacks matching the package name
langpacks = [ i for i in self.langpacks if i["name"] == po.name ]
if not langpacks:
continue
for langpack in langpacks:
pattern = langpack["install"] % "*" # replace '%s' with '*'
exactmatched, matched, unmatched = yum.packages.parsePackages(self.pkgs, [pattern], casematch=1, pkgdict=self.pkg_refs.copy())
matches = filter(self._filtersrcdebug, exactmatched + matched)
matches = [ i for i in matches if not i.name.endswith("-devel") and not i.name.endswith("-static") and i.name != "man-pages-overrides" ]
matches = [ i for i in matches if fnmatch(i.name, pattern) ]
for langpack in langpacks:
pattern = langpack["install"] % "*" # replace '%s' with '*'
exactmatched, matched, unmatched = yum.packages.parsePackages(self.pkgs, [pattern], casematch=1, pkgdict=self.pkg_refs.copy())
matches = filter(self._filtersrcdebug, exactmatched + matched)
matches = [ i for i in matches if not i.name.endswith("-devel") and not i.name.endswith("-static") and i.name != "man-pages-overrides" ]
matches = [ i for i in matches if fnmatch(i.name, pattern) ]
packages_by_name = {}
for i in matches:
packages_by_name.setdefault(i.name, []).append(i)
packages_by_name = {}
for i in matches:
packages_by_name.setdefault(i.name, []).append(i)
for i, pkg_sack in packages_by_name.iteritems():
pkg_sack = self.excludePackages(pkg_sack)
match = self.ayum._bestPackageFromList(pkg_sack)
msg = 'Added langpack %s.%s for package %s (pattern: %s)' % (match.name, match.arch, po.name, pattern)
self.add_package(match, msg)
added.append(match)
for i, pkg_sack in packages_by_name.iteritems():
pkg_sack = self.excludePackages(pkg_sack)
match = self.ayum._bestPackageFromList(pkg_sack)
msg = 'Added langpack %s.%s for package %s (pattern: %s)' % (match.name, match.arch, po.name, pattern)
self.add_package(match, msg)
added.append(match)
return added
def getMultilib(self, po_list):
added = []
if not self.multilib_methods:
return added
for po in po_list:
if po.arch in ("noarch", "src", "nosrc"):
continue
if po.arch in self.valid_multilib_arches:
continue
matches = self.ayum.pkgSack.searchNevra(name=po.name, ver=po.version, rel=po.release)
matches = [i for i in matches if i.arch in self.valid_multilib_arches]
if not matches:
continue
matches = self.excludePackages(matches)
match = self.ayum._bestPackageFromList(matches)
if not match:
continue
method = multilib.po_is_multilib(po, self.multilib_methods)
if not method:
continue
msg = "Added multilib package %s.%s for package %s.%s (method: %s)" % (match.name, match.arch, po.name, po.arch, method)
self.add_package(match, msg)
added.append(match)
return added
def getPackagesFromGroup(self, group):
"""Get a list of package names from a ksparser group object

1
src/pypungi/config.py
View File

@ -46,5 +46,6 @@ class Config(SafeConfigParser):
self.set('pungi', 'isfinal', "False")
self.set('pungi', 'nohash', "False")
self.set('pungi', 'full_archlist', "False")
self.set('pungi', 'multilib', '')
self.set('pungi', 'lookaside_repos', '')

371
src/pypungi/multilib.py Executable file
View File

@ -0,0 +1,371 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import re
import fnmatch
import pathmatch
#from pypungi import is_package, is_source, is_debug, is_noarch
import pypungi
LINE_PATTERN_RE = re.compile(r"^\s*(?P<line>[^#]+)(:?\s+(?P<comment>#.*))?$")
RUNTIME_PATTERN_SPLIT_RE = re.compile(r"^\s*(?P<path>[^\s]+)\s+(?P<pattern>[^\s]+)(:?\s+(?P<comment>#.*))?$")
def read_lines(lines):
result = []
for i in lines:
i = i.strip()
if not i:
continue
# skip comments
if i.startswith("#"):
continue
match = LINE_PATTERN_RE.match(i)
if match is None:
raise ValueError("Couldn't parse line: %s" % i)
gd = match.groupdict()
result.append(gd["line"])
return result
def read_lines_from_file(path):
lines = open(path, "r").readlines()
lines = read_lines(lines)
return lines
def read_runtime_patterns(lines):
result = []
for i in read_lines(lines):
match = RUNTIME_PATTERN_SPLIT_RE.match(i)
if match is None:
raise ValueError("Couldn't parse pattern: %s" % i)
gd = match.groupdict()
result.append((gd["path"], gd["pattern"]))
return result
def read_runtime_patterns_from_file(path):
lines = open(path, "r").readlines()
return read_runtime_patterns(lines)
def expand_runtime_patterns(patterns):
pm = pathmatch.PathMatch()
result = []
for path, pattern in patterns:
for root in ("", "/opt/*/*/root"):
# include Software Collections: /opt/<vendor>/<scl_name>/root/...
if "$LIBDIR" in path:
for lib_dir in ("/lib", "/lib64", "/usr/lib", "/usr/lib64"):
path_pattern = path.replace("$LIBDIR", lib_dir)
path_pattern = "%s/%s" % (root, path_pattern.lstrip("/"))
pm[path_pattern] = (path_pattern, pattern)
else:
path_pattern = "%s/%s" % (root, path.lstrip("/"))
pm[path_pattern] = (path_pattern, pattern)
return pm
class MultilibMethodBase(object):
"""a base class for multilib methods"""
name = "base"
def select(self, po):
raise NotImplementedError
def skip(self, po):
if pypungi.is_noarch(po) or pypungi.is_source(po) or pypungi.is_debug(po):
return True
return False
def is_kernel(self, po):
for p_name, p_flag, (p_e, p_v, p_r) in po.provides:
if p_name == "kernel":
return True
return False
def is_kernel_devel(self, po):
for p_name, p_flag, (p_e, p_v, p_r) in po.provides:
if p_name == "kernel-devel":
return True
return False
def is_kernel_or_kernel_devel(self, po):
for p_name, p_flag, (p_e, p_v, p_r) in po.provides:
if p_name in ("kernel", "kernel-devel"):
return True
return False
class NoneMultilibMethod(MultilibMethodBase):
"""multilib disabled"""
name = "none"
def select(self, po):
return False
class AllMultilibMethod(MultilibMethodBase):
"""all packages are multilib"""
name = "all"
def select(self, po):
if self.skip(po):
return False
return True
class RuntimeMultilibMethod(MultilibMethodBase):
"""pre-defined paths to libs"""
name = "runtime"
def __init__(self, **kwargs):
self.blacklist = read_lines_from_file("/usr/share/pungi/multilib/runtime-blacklist.conf")
self.whitelist = read_lines_from_file("/usr/share/pungi/multilib/runtime-whitelist.conf")
self.patterns = expand_runtime_patterns(read_runtime_patterns_from_file("/usr/share/pungi/multilib/runtime-patterns.conf"))
def select(self, po):
if self.skip(po):
return False
if po.name in self.blacklist:
return False
if po.name in self.whitelist:
return True
if self.is_kernel(po):
return False
for path in po.returnFileEntries() + po.returnFileEntries("ghost"):
dirname, filename = path.rsplit("/", 1)
dirname = dirname.rstrip("/")
patterns = self.patterns[dirname]
if not patterns:
continue
for dir_pattern, file_pattern in patterns:
if file_pattern == "-":
return True
if fnmatch.fnmatch(filename, file_pattern):
return True
return False
class FileMultilibMethod(MultilibMethodBase):
"""explicitely defined whitelist and blacklist"""
def __init__(self, **kwargs):
self.name = "file"
whitelist = kwargs.pop("whitelist", None)
blacklist = kwargs.pop("blacklist", None)
self.whitelist = self.read_file(whitelist)
self.blacklist = self.read_file(blacklist)
@staticmethod
def read_file(path):
if not path:
return []
result = [ i.strip() for i in open(path, "r") if not i.strip().startswith("#") ]
return result
def select(self, po):
for pattern in self.blacklist:
if fnmatch.fnmatch(po.name, pattern):
return False
for pattern in self.whitelist:
if fnmatch.fnmatch(po.name, pattern):
return False
return False
class KernelMultilibMethod(MultilibMethodBase):
"""kernel and kernel-devel"""
def __init__(self, **kwargs):
self.name = "kernel"
def select(self, po):
if self.is_kernel_or_kernel_devel(po):
return True
return False
class DevelMultilibMethod(MultilibMethodBase):
"""all -devel and -static packages"""
name = "devel"
def __init__(self, **kwargs):
self.blacklist = read_lines_from_file("/usr/share/pungi/multilib/devel-blacklist.conf")
self.whitelist = read_lines_from_file("/usr/share/pungi/multilib/devel-whitelist.conf")
def select(self, po):
if self.skip(po):
return False
if po.name in self.blacklist:
return False
if po.name in self.whitelist:
return True
if self.is_kernel_devel(po):
return False
# HACK: exclude ghc*
if po.name.startswith("ghc-"):
return False
if po.name.endswith("-devel"):
return True
if po.name.endswith("-static"):
return True
for p_name, p_flag, (p_e, p_v, p_r) in po.provides:
if p_name.endswith("-devel"):
return True
if p_name.endswith("-static"):
return True
return False
DEFAULT_METHODS = ["devel", "runtime"]
METHOD_MAP = {}
for cls in (AllMultilibMethod, DevelMultilibMethod, FileMultilibMethod, KernelMultilibMethod, NoneMultilibMethod, RuntimeMultilibMethod):
method = cls()
METHOD_MAP[method.name] = method
def po_is_multilib(po, methods):
for method_name in methods:
if not method_name:
continue
method = METHOD_MAP[method_name]
if method.select(po):
return method_name
return None
def do_multilib(yum_arch, methods, repos, tmpdir, logfile):
import os
import yum
import rpm
import logging
archlist = yum.rpmUtils.arch.getArchList(yum_arch)
yumbase = yum.YumBase()
yumbase.preconf.init_plugins = False
yumbase.preconf.root = tmpdir
# order matters!
# must run doConfigSetup() before touching yumbase.conf
yumbase.doConfigSetup(fn="/dev/null")
yumbase.conf.cache = False
yumbase.conf.cachedir = tmpdir
yumbase.conf.exactarch = True
yumbase.conf.gpgcheck = False
yumbase.conf.logfile = logfile
yumbase.conf.plugins = False
yumbase.conf.reposdir = []
yumbase.verbose_logger.setLevel(logging.ERROR)
yumbase.doRepoSetup()
yumbase.doTsSetup()
yumbase.doRpmDBSetup()
yumbase.ts.pushVSFlags((rpm._RPMVSF_NOSIGNATURES|rpm._RPMVSF_NODIGESTS))
for repo in yumbase.repos.findRepos("*"):
repo.disable()
for i, baseurl in enumerate(repos):
repo_id = "multilib-%s" % i
if "://" not in baseurl:
baseurl = "file://" + os.path.abspath(baseurl)
yumbase.add_enable_repo(repo_id, baseurls=[baseurl])
yumbase.doSackSetup(archlist=archlist)
yumbase.doSackFilelistPopulate()
method_kwargs = {}
result = []
for po in sorted(yumbase.pkgSack):
method = po_is_multilib(po, methods)
if method:
nvra = "%s-%s-%s.%s.rpm" % (po.name, po.version, po.release, po.arch)
result.append((nvra, method))
return result
def main():
import optparse
import shutil
import tempfile
class MyOptionParser(optparse.OptionParser):
def print_help(self, *args, **kwargs):
optparse.OptionParser.print_help(self, *args, **kwargs)
print
print "Available multilib methods:"
for key, value in sorted(METHOD_MAP.items()):
default = (key in DEFAULT_METHODS) and " (default)" or ""
print " %-10s %s%s" % (key, value.__doc__ or "", default)
parser = MyOptionParser("usage: %prog [options]")
parser.add_option(
"--arch",
)
parser.add_option(
"--method",
action="append",
default=DEFAULT_METHODS,
help="multilib method",
)
parser.add_option(
"--repo",
dest="repos",
action="append",
help="path or url to yum repo; can be specified multiple times",
)
parser.add_option("--tmpdir")
parser.add_option("--logfile", action="store")
opts, args = parser.parse_args()
if args:
parser.error("no arguments expected")
if not opts.repos:
parser.error("provide at least one repo")
for method_name in opts.method:
if method_name not in METHOD_MAP:
parser.error("unknown method: %s" % method_name)
print opts.method
tmpdir = opts.tmpdir
if not opts.tmpdir:
tmpdir = tempfile.mkdtemp(prefix="multilib_")
nvra_list = do_multilib(opts.arch, opts.method, opts.repos, tmpdir, opts.logfile)
for nvra, method in nvra_list:
print "MULTILIB(%s): %s" % (method, nvra)
if not opts.tmpdir:
shutil.rmtree(tmpdir)
if __name__ == "__main__":
main()

60
src/pypungi/pathmatch.py Normal file
View File

@ -0,0 +1,60 @@
# -*- coding: utf-8 -*-
import fnmatch
def head_tail_split(name):
name_split = name.strip("/").split("/", 1)
if len(name_split) == 2:
head = name_split[0]
tail = name_split[1].strip("/")
else:
head, tail = name_split[0], None
return head, tail
class PathMatch(object):
def __init__(self, parent=None, desc=None):
self._patterns = {}
self._final_patterns = {}
self._values = []
def __setitem__(self, name, value):
head, tail = head_tail_split(name)
if tail is not None:
# recursion
if head not in self._patterns:
self._patterns[head] = PathMatch(parent=self, desc=head)
self._patterns[head][tail] = value
else:
if head not in self._final_patterns:
self._final_patterns[head] = PathMatch(parent=self, desc=head)
if value not in self._final_patterns[head]._values:
self._final_patterns[head]._values.append(value)
def __getitem__(self, name):
result = []
head, tail = head_tail_split(name)
for pattern in self._patterns:
if fnmatch.fnmatch(head, pattern):
if tail is None:
values = self._patterns[pattern]._values
else:
values = self._patterns[pattern][tail]
for value in values:
if value not in result:
result.append(value)
for pattern in self._final_patterns:
if tail is None:
x = head
else:
x = "%s/%s" % (head, tail)
if fnmatch.fnmatch(x, pattern):
values = self._final_patterns[pattern]._values
for value in values:
if value not in result:
result.append(value)
return result

71
tests/test_pathmatch.py Executable file
View File

@ -0,0 +1,71 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
import unittest
import os
import sys
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "src", "pypungi")))
from pathmatch import PathMatch, head_tail_split
class TestHeadTailSplit(unittest.TestCase):
def test_1(self):
head, tail = head_tail_split("a")
self.assertEqual(head, "a")
self.assertEqual(tail, None)
head, tail = head_tail_split("/*")
self.assertEqual(head, "*")
self.assertEqual(tail, None)
head, tail = head_tail_split("///*")
self.assertEqual(head, "*")
self.assertEqual(tail, None)
head, tail = head_tail_split("///*//")
self.assertEqual(head, "*")
self.assertEqual(tail, None)
head, tail = head_tail_split("///*//-")
self.assertEqual(head, "*")
self.assertEqual(tail, "-")
class TestPathMatch(unittest.TestCase):
def setUp(self):
self.pm = PathMatch()
def test_1(self):
self.pm["/*"] = "/star1"
self.assertEqual(self.pm._final_patterns.keys(), ["*"])
self.assertEqual(self.pm._values, [])
self.assertEqual(self.pm._final_patterns["*"]._values, ["/star1"])
self.assertEqual(sorted(self.pm["/lib"]), ["/star1"])
self.pm["/*"] = "/star2"
self.assertEqual(sorted(self.pm["/lib"]), ["/star1", "/star2"])
self.pm["/lib"] = "/lib"
self.assertEqual(sorted(self.pm["/lib"]), ["/lib", "/star1", "/star2"])
self.pm["/lib64"] = "/lib64"
self.assertEqual(sorted(self.pm["/lib64"]), ["/lib64", "/star1", "/star2"])
def test_2(self):
self.pm["/*/*"] = "/star/star1"
self.assertEqual(self.pm._patterns.keys(), ["*"])
self.assertEqual(self.pm._patterns["*"]._final_patterns.keys(), ["*"])
self.assertEqual(self.pm._patterns["*"]._final_patterns["*"]._values, ["/star/star1"])
self.assertEqual(sorted(self.pm["/lib/asd"]), ["/star/star1"])
self.pm["/*"] = "/star2"
self.assertEqual(sorted(self.pm["/lib"]), ["/star2"])
self.assertEqual(sorted(self.pm["/lib/foo"]), ["/star/star1", "/star2"])
if __name__ == "__main__":
unittest.main()