From 0a3e5b27bf60a329ee0bff05c7a9bbaf34745aa9 Mon Sep 17 00:00:00 2001 From: Qixiang Wan Date: Tue, 21 Mar 2017 12:32:53 +0800 Subject: [PATCH] remove the dependency of rpmUtils The rpmUtils module is provided yum-utils package, which is only available for Python 2. There is no replacement for the functionality in DNF. There is a proposal to add this functionality to rpm itself, but it's not really moving forward very much: https://bugzilla.redhat.com/show_bug.cgi?id=1072972 As a short term solution let's copy the needed parts of rpmUtils.arch module directly to pungi code base. Fixes: https://pagure.io/pungi/issue/533 Signed-off-by: Qixiang Wan --- doc/contributing.rst | 2 +- pungi/arch.py | 17 +- pungi/arch_utils.py | 372 ++++++++++++++++++++++++++++++++++ pungi/config.py | 5 +- pungi/ostree/utils.py | 6 +- pungi/wrappers/kojiwrapper.py | 6 +- tests/test_arch.py | 2 +- 7 files changed, 390 insertions(+), 20 deletions(-) create mode 100644 pungi/arch_utils.py diff --git a/doc/contributing.rst b/doc/contributing.rst index 0bef6f04..2bef5b60 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -46,7 +46,7 @@ PyPI or from tarball. You will still need to install all of the non-Python packages above as they are used by calling an executable. :: $ mkvirtualenv pungienv - $ for pkg in koji rpm rpmUtils pykickstart selinux createrepo yum urlgrabber; do ln -vs "$(deactivate && python -c 'import os, '$pkg'; print os.path.dirname('$pkg'.__file__)')" "$(virtualenvwrapper_get_site_packages_dir)"; done + $ for pkg in koji rpm pykickstart selinux createrepo yum urlgrabber; do ln -vs "$(deactivate && python -c 'import os, '$pkg'; print os.path.dirname('$pkg'.__file__)')" "$(virtualenvwrapper_get_site_packages_dir)"; done $ for pkg in _selinux deltarpm _deltarpm krbV sqlitecachec _sqlitecache; do ln -vs "$(deactivate && python -c 'import os, '$pkg'; print '$pkg'.__file__')" "$(virtualenvwrapper_get_site_packages_dir)"; done $ PYCURL_SSL_LIBRARY=nss pip install pycurl --no-binary :all: $ pip install lxml pyopenssl mock sphinx setuptools nose nose-cov productmd jsonschema requests lockfile python-multilib kobo diff --git a/pungi/arch.py b/pungi/arch.py index e24afdcb..ab121d99 100644 --- a/pungi/arch.py +++ b/pungi/arch.py @@ -13,9 +13,8 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, see . - -import rpmUtils.arch - +from .arch_utils import arches as ALL_ARCHES +from .arch_utils import getBaseArch, getMultiArchInfo, getArchList TREE_ARCH_YUM_ARCH_MAP = { "i386": "athlon", @@ -27,13 +26,13 @@ TREE_ARCH_YUM_ARCH_MAP = { def tree_arch_to_yum_arch(tree_arch): - # this is basically an opposite to rpmUtils.arch.getBaseArch() + # this is basically an opposite to pungi.arch_utils.getBaseArch() yum_arch = TREE_ARCH_YUM_ARCH_MAP.get(tree_arch, tree_arch) return yum_arch def get_multilib_arch(yum_arch): - arch_info = rpmUtils.arch.getMultiArchInfo(yum_arch) + arch_info = getMultiArchInfo(yum_arch) if arch_info is None: return None return arch_info[0] @@ -44,14 +43,14 @@ def get_valid_multilib_arches(tree_arch): multilib_arch = get_multilib_arch(yum_arch) if not multilib_arch: return [] - return [i for i in rpmUtils.arch.getArchList(multilib_arch) if i not in ("noarch", "src")] + return [i for i in getArchList(multilib_arch) if i not in ("noarch", "src")] def get_valid_arches(tree_arch, multilib=True, add_noarch=True, add_src=False): result = [] yum_arch = tree_arch_to_yum_arch(tree_arch) - for arch in rpmUtils.arch.getArchList(yum_arch): + for arch in getArchList(yum_arch): if arch not in result: result.append(arch) @@ -70,7 +69,7 @@ def get_valid_arches(tree_arch, multilib=True, add_noarch=True, add_src=False): def get_compatible_arches(arch, multilib=False): - tree_arch = rpmUtils.arch.getBaseArch(arch) + tree_arch = getBaseArch(arch) compatible_arches = get_valid_arches(tree_arch, multilib=multilib) return compatible_arches @@ -78,7 +77,7 @@ def get_compatible_arches(arch, multilib=False): def is_valid_arch(arch): if arch in ("noarch", "src", "nosrc"): return True - if arch in rpmUtils.arch.arches: + if arch in ALL_ARCHES: return True return False diff --git a/pungi/arch_utils.py b/pungi/arch_utils.py new file mode 100644 index 00000000..b28179a0 --- /dev/null +++ b/pungi/arch_utils.py @@ -0,0 +1,372 @@ +#!/usr/bin/python + +# A copy of some necessary parts from yum.rpmUtils.arch, with slightly changes: +# 1. _ppc64_native_is_best changed to True +# 2. code style fixes for flake8 reported errors + +import os +import rpm +import ctypes +import struct + +# _ppc64_native_is_best is False in yum's source code, but patched with a +# separate patch when built from source rpm, so we set it to True here. +_ppc64_native_is_best = True + +# dict mapping arch -> ( multicompat, best personality, biarch personality ) +multilibArches = {"x86_64": ("athlon", "x86_64", "athlon"), + "sparc64v": ("sparcv9v", "sparcv9v", "sparc64v"), + "sparc64": ("sparcv9", "sparcv9", "sparc64"), + "ppc64": ("ppc", "ppc", "ppc64"), + "s390x": ("s390", "s390x", "s390"), + } +if _ppc64_native_is_best: + multilibArches["ppc64"] = ("ppc", "ppc64", "ppc64") + +arches = { + # ia32 + "athlon": "i686", + "i686": "i586", + "geode": "i586", + "i586": "i486", + "i486": "i386", + "i386": "noarch", + + # amd64 + "x86_64": "athlon", + "amd64": "x86_64", + "ia32e": "x86_64", + + # ppc64le + "ppc64le": "noarch", + + # ppc + "ppc64p7": "ppc64", + "ppc64pseries": "ppc64", + "ppc64iseries": "ppc64", + "ppc64": "ppc", + "ppc": "noarch", + + # s390{,x} + "s390x": "s390", + "s390": "noarch", + + # sparc + "sparc64v": "sparcv9v", + "sparc64": "sparcv9", + "sparcv9v": "sparcv9", + "sparcv9": "sparcv8", + "sparcv8": "sparc", + "sparc": "noarch", + + # alpha + "alphaev7": "alphaev68", + "alphaev68": "alphaev67", + "alphaev67": "alphaev6", + "alphaev6": "alphapca56", + "alphapca56": "alphaev56", + "alphaev56": "alphaev5", + "alphaev5": "alphaev45", + "alphaev45": "alphaev4", + "alphaev4": "alpha", + "alpha": "noarch", + + # arm + "armv7l": "armv6l", + "armv6l": "armv5tejl", + "armv5tejl": "armv5tel", + "armv5tel": "noarch", + + # arm hardware floating point + "armv7hnl": "armv7hl", + "armv7hl": "armv6hl", + "armv6hl": "noarch", + + # arm64 + "arm64": "noarch", + + # aarch64 + "aarch64": "noarch", + + # super-h + "sh4a": "sh4", + "sh4": "noarch", + "sh3": "noarch", + + # itanium + "ia64": "noarch", +} + +# Will contain information parsed from /proc/self/auxv via _parse_auxv(). +# Should move into rpm really. +_aux_vector = { + "platform": "", + "hwcap": 0, +} + + +def isMultiLibArch(arch=None): # pragma: no cover + """returns true if arch is a multilib arch, false if not""" + if arch is None: + arch = canonArch + + if arch not in arches: # or we could check if it is noarch + return 0 + + if arch in multilibArches: + return 1 + + if arches[arch] in multilibArches: + return 1 + + return 0 + + +def getArchList(thisarch=None): # pragma: no cover + # this returns a list of archs that are compatible with arch given + if not thisarch: + thisarch = canonArch + + archlist = [thisarch] + while thisarch in arches: + thisarch = arches[thisarch] + archlist.append(thisarch) + + # hack hack hack + # sparc64v is also sparc64 compat + if archlist[0] == "sparc64v": + archlist.insert(1, "sparc64") + + # if we're a weirdo arch - add noarch on there. + if len(archlist) == 1 and archlist[0] == thisarch: + archlist.append('noarch') + return archlist + + +def _try_read_cpuinfo(): # pragma: no cover + """ Try to read /proc/cpuinfo ... if we can't ignore errors (ie. proc not + mounted). """ + try: + return open("/proc/cpuinfo", "r") + except: + return [] + + +def _parse_auxv(): # pragma: no cover + """ Read /proc/self/auxv and parse it into global dict for easier access + later on, very similar to what rpm does. """ + # In case we can't open and read /proc/self/auxv, just return + try: + data = open("/proc/self/auxv", "rb").read() + except: + return + + # Define values from /usr/include/elf.h + AT_PLATFORM = 15 + AT_HWCAP = 16 + fmtlen = struct.calcsize("LL") + offset = 0 + platform = ctypes.c_char_p() + + # Parse the data and fill in _aux_vector dict + while offset <= len(data) - fmtlen: + at_type, at_val = struct.unpack_from("LL", data, offset) + if at_type == AT_PLATFORM: + platform.value = at_val + _aux_vector["platform"] = platform.value + if at_type == AT_HWCAP: + _aux_vector["hwcap"] = at_val + offset = offset + fmtlen + + +def getCanonX86Arch(arch): # pragma: no cover + if arch == "i586": + for line in _try_read_cpuinfo(): + if line.startswith("model name"): + if line.find("Geode(TM)") != -1: + return "geode" + break + return arch + # only athlon vs i686 isn't handled with uname currently + if arch != "i686": + return arch + + # if we're i686 and AuthenticAMD, then we should be an athlon + for line in _try_read_cpuinfo(): + if line.startswith("vendor") and line.find("AuthenticAMD") != -1: + return "athlon" + # i686 doesn't guarantee cmov, but we depend on it + elif line.startswith("flags"): + if line.find("cmov") == -1: + return "i586" + break + + return arch + + +def getCanonARMArch(arch): # pragma: no cover + # the %{_target_arch} macro in rpm will let us know the abi we are using + target = rpm.expandMacro('%{_target_cpu}') + if target.startswith('armv6h'): + return target + if target.startswith('armv7h'): + return target + return arch + + +def getCanonPPCArch(arch): # pragma: no cover + # FIXME: should I do better handling for mac, etc? + if arch != "ppc64": + return arch + + machine = None + for line in _try_read_cpuinfo(): + if line.find("machine") != -1: + machine = line.split(':')[1] + break + + platform = _aux_vector["platform"] + if machine is None and not platform: + return arch + + try: + if platform.startswith("power") and int(platform[5:].rstrip('+')) >= 7: + return "ppc64p7" + except: + pass + + if machine is None: + return arch + + if machine.find("CHRP IBM") != -1: + return "ppc64pseries" + if machine.find("iSeries") != -1: + return "ppc64iseries" + return arch + + +def getCanonSPARCArch(arch): # pragma: no cover + # Deal with sun4v, sun4u, sun4m cases + SPARCtype = None + for line in _try_read_cpuinfo(): + if line.startswith("type"): + SPARCtype = line.split(':')[1] + break + if SPARCtype is None: + return arch + + if SPARCtype.find("sun4v") != -1: + if arch.startswith("sparc64"): + return "sparc64v" + else: + return "sparcv9v" + if SPARCtype.find("sun4u") != -1: + if arch.startswith("sparc64"): + return "sparc64" + else: + return "sparcv9" + if SPARCtype.find("sun4m") != -1: + return "sparcv8" + return arch + + +def getCanonX86_64Arch(arch): # pragma: no cover + if arch != "x86_64": + return arch + + vendor = None + for line in _try_read_cpuinfo(): + if line.startswith("vendor_id"): + vendor = line.split(':')[1] + break + if vendor is None: + return arch + + if vendor.find("Authentic AMD") != -1 or vendor.find("AuthenticAMD") != -1: + return "amd64" + if vendor.find("GenuineIntel") != -1: + return "ia32e" + return arch + + +def getCanonArch(skipRpmPlatform=0): # pragma: no cover + if not skipRpmPlatform and os.access("/etc/rpm/platform", os.R_OK): + try: + f = open("/etc/rpm/platform", "r") + line = f.readline() + f.close() + (arch, vendor, opersys) = line.split("-", 2) + return arch + except: + pass + + arch = os.uname()[4] + + _parse_auxv() + + if (len(arch) == 4 and arch[0] == "i" and arch[2:4] == "86"): + return getCanonX86Arch(arch) + + if arch.startswith("arm"): + return getCanonARMArch(arch) + if arch.startswith("ppc"): + return getCanonPPCArch(arch) + if arch.startswith("sparc"): + return getCanonSPARCArch(arch) + if arch == "x86_64": + return getCanonX86_64Arch(arch) + + return arch + +canonArch = getCanonArch() + + +# this gets you the "compat" arch of a biarch pair +def getMultiArchInfo(arch=canonArch): # pragma: no cover + if arch in multilibArches: + return multilibArches[arch] + if arch in arches and arches[arch] != "noarch": + return getMultiArchInfo(arch=arches[arch]) + return None + + +def getBaseArch(myarch=None): # pragma: no cover + """returns 'base' arch for myarch, if specified, or canonArch if not. + base arch is the arch before noarch in the arches dict if myarch is not + a key in the multilibArches.""" + + if not myarch: + myarch = canonArch + + if myarch not in arches: # this is dumb, but + return myarch + + if myarch.startswith("sparc64"): + return "sparc" + elif myarch == "ppc64le": + return "ppc64le" + elif myarch.startswith("ppc64") and not _ppc64_native_is_best: + return "ppc" + elif myarch.startswith("arm64"): + return "arm64" + elif myarch.startswith("armv6h"): + return "armhfp" + elif myarch.startswith("armv7h"): + return "armhfp" + elif myarch.startswith("arm"): + return "arm" + + if isMultiLibArch(arch=myarch): + if myarch in multilibArches: + return myarch + else: + return arches[myarch] + + if myarch in arches: + basearch = myarch + value = arches[basearch] + while value != 'noarch': + basearch = value + value = arches[basearch] + + return basearch diff --git a/pungi/config.py b/pungi/config.py index 88ece6b7..7fd783e0 100644 --- a/pungi/config.py +++ b/pungi/config.py @@ -17,11 +17,10 @@ import os import sys import time -import yum from ConfigParser import SafeConfigParser - +from .arch_utils import getBaseArch # In development, `here` will point to the bin/ directory with scripts. here = sys.path[0] MULTILIBCONF = (os.path.join(os.path.dirname(__file__), '..', 'share', 'multilib') @@ -47,7 +46,7 @@ class Config(SafeConfigParser): self.set('pungi', 'product_path', 'Packages') self.set('pungi', 'cachedir', '/var/cache/pungi') self.set('pungi', 'compress_type', 'xz') - self.set('pungi', 'arch', yum.rpmUtils.arch.getBaseArch()) + self.set('pungi', 'arch', getBaseArch()) self.set('pungi', 'family', 'Fedora') self.set('pungi', 'iso_basename', 'Fedora') self.set('pungi', 'version', time.strftime('%Y%m%d', time.localtime())) diff --git a/pungi/ostree/utils.py b/pungi/ostree/utils.py index 23663cea..c832df3b 100644 --- a/pungi/ostree/utils.py +++ b/pungi/ostree/utils.py @@ -18,8 +18,8 @@ import datetime import json import logging import os -import rpmUtils.arch +from pungi.arch_utils import getBaseArch from pungi.util import makedirs @@ -43,9 +43,9 @@ def get_ref_from_treefile(treefile, arch=None, logger=None): try: parsed = json.loads(f.read()) if arch is None: - basearch = rpmUtils.arch.getBaseArch() + basearch = getBaseArch() else: - basearch = rpmUtils.arch.getBaseArch(arch) + basearch = getBaseArch(arch) ref = parsed['ref'].replace('${basearch}', basearch) except Exception as e: logger.error('Unable to get ref from treefile: %s' % e) diff --git a/pungi/wrappers/kojiwrapper.py b/pungi/wrappers/kojiwrapper.py index b0744bc1..aa08f59d 100644 --- a/pungi/wrappers/kojiwrapper.py +++ b/pungi/wrappers/kojiwrapper.py @@ -22,11 +22,11 @@ import threading import contextlib import koji -import rpmUtils.arch from kobo.shortcuts import run from ConfigParser import ConfigParser from .. import util +from ..arch_utils import getBaseArch class KojiWrapper(object): @@ -100,7 +100,7 @@ class KojiWrapper(object): cmd.append(target) # i686 -> i386 etc. - arch = rpmUtils.arch.getBaseArch(arch) + arch = getBaseArch(arch) cmd.append(arch) if isinstance(command, list): @@ -280,7 +280,7 @@ class KojiWrapper(object): cmd.append(target) # i686 -> i386 etc. - arch = rpmUtils.arch.getBaseArch(arch) + arch = getBaseArch(arch) cmd.append(arch) cmd.append(ks_file) diff --git a/tests/test_arch.py b/tests/test_arch.py index c5edd818..dbef1aad 100644 --- a/tests/test_arch.py +++ b/tests/test_arch.py @@ -56,7 +56,7 @@ class MockArchModule(object): return ARCHES[yum_arch] -@mock.patch('rpmUtils.arch', MockArchModule) +@mock.patch('pungi.arch_utils', MockArchModule) class TestArch(unittest.TestCase): def test_i386(self):