remove the dependency of rpmUtils

The rpmUtils module is provided yum-utils package, which is only
available for Python 2. There is no replacement for the functionality in
DNF.

There is a proposal to add this functionality to rpm itself, but it's
not really moving forward very much:
https://bugzilla.redhat.com/show_bug.cgi?id=1072972

As a short term solution let's copy the needed parts of rpmUtils.arch
module directly to pungi code base.

Fixes: https://pagure.io/pungi/issue/533
Signed-off-by: Qixiang Wan <qwan@redhat.com>

doc/contributing.rst

@@ -46,7 +46,7 @@ PyPI or from tarball. You will still need to install all of the non-Python
 packages above as they are used by calling an executable. ::
 
     $ mkvirtualenv pungienv
-    $ for pkg in koji rpm rpmUtils pykickstart selinux createrepo yum urlgrabber; do ln -vs "$(deactivate && python -c 'import os, '$pkg'; print os.path.dirname('$pkg'.__file__)')" "$(virtualenvwrapper_get_site_packages_dir)"; done
+    $ for pkg in koji rpm pykickstart selinux createrepo yum urlgrabber; do ln -vs "$(deactivate && python -c 'import os, '$pkg'; print os.path.dirname('$pkg'.__file__)')" "$(virtualenvwrapper_get_site_packages_dir)"; done
     $ for pkg in _selinux deltarpm _deltarpm krbV sqlitecachec _sqlitecache; do ln -vs "$(deactivate && python -c 'import os, '$pkg'; print '$pkg'.__file__')" "$(virtualenvwrapper_get_site_packages_dir)"; done
     $ PYCURL_SSL_LIBRARY=nss pip install pycurl --no-binary :all:
     $ pip install lxml pyopenssl mock sphinx setuptools nose nose-cov productmd jsonschema requests lockfile python-multilib kobo

pungi/arch.py

@@ -13,9 +13,8 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, see <https://gnu.org/licenses/>.
 
-
-import rpmUtils.arch
-
+from .arch_utils import arches as ALL_ARCHES
+from .arch_utils import getBaseArch, getMultiArchInfo, getArchList
 
 TREE_ARCH_YUM_ARCH_MAP = {
     "i386": "athlon",
@@ -27,13 +26,13 @@ TREE_ARCH_YUM_ARCH_MAP = {
 
 
 def tree_arch_to_yum_arch(tree_arch):
-    # this is basically an opposite to rpmUtils.arch.getBaseArch()
+    # this is basically an opposite to pungi.arch_utils.getBaseArch()
     yum_arch = TREE_ARCH_YUM_ARCH_MAP.get(tree_arch, tree_arch)
     return yum_arch
 
 
 def get_multilib_arch(yum_arch):
-    arch_info = rpmUtils.arch.getMultiArchInfo(yum_arch)
+    arch_info = getMultiArchInfo(yum_arch)
     if arch_info is None:
         return None
     return arch_info[0]
@@ -44,14 +43,14 @@ def get_valid_multilib_arches(tree_arch):
     multilib_arch = get_multilib_arch(yum_arch)
     if not multilib_arch:
         return []
-    return [i for i in rpmUtils.arch.getArchList(multilib_arch) if i not in ("noarch", "src")]
+    return [i for i in getArchList(multilib_arch) if i not in ("noarch", "src")]
 
 
 def get_valid_arches(tree_arch, multilib=True, add_noarch=True, add_src=False):
     result = []
 
     yum_arch = tree_arch_to_yum_arch(tree_arch)
-    for arch in rpmUtils.arch.getArchList(yum_arch):
+    for arch in getArchList(yum_arch):
         if arch not in result:
             result.append(arch)
 
@@ -70,7 +69,7 @@ def get_valid_arches(tree_arch, multilib=True, add_noarch=True, add_src=False):
 
 
 def get_compatible_arches(arch, multilib=False):
-    tree_arch = rpmUtils.arch.getBaseArch(arch)
+    tree_arch = getBaseArch(arch)
     compatible_arches = get_valid_arches(tree_arch, multilib=multilib)
     return compatible_arches
 
@@ -78,7 +77,7 @@ def get_compatible_arches(arch, multilib=False):
 def is_valid_arch(arch):
     if arch in ("noarch", "src", "nosrc"):
         return True
-    if arch in rpmUtils.arch.arches:
+    if arch in ALL_ARCHES:
         return True
     return False
 

pungi/arch_utils.py

@@ -0,0 +1,372 @@
+#!/usr/bin/python
+
+# A copy of some necessary parts from yum.rpmUtils.arch, with slightly changes:
+# 1. _ppc64_native_is_best changed to True
+# 2. code style fixes for flake8 reported errors
+
+import os
+import rpm
+import ctypes
+import struct
+
+# _ppc64_native_is_best is False in yum's source code, but patched with a
+# separate patch when built from source rpm, so we set it to True here.
+_ppc64_native_is_best = True
+
+# dict mapping arch -> ( multicompat, best personality, biarch personality )
+multilibArches = {"x86_64": ("athlon", "x86_64", "athlon"),
+                  "sparc64v": ("sparcv9v", "sparcv9v", "sparc64v"),
+                  "sparc64": ("sparcv9", "sparcv9", "sparc64"),
+                  "ppc64": ("ppc", "ppc", "ppc64"),
+                  "s390x": ("s390", "s390x", "s390"),
+                  }
+if _ppc64_native_is_best:
+    multilibArches["ppc64"] = ("ppc", "ppc64", "ppc64")
+
+arches = {
+    # ia32
+    "athlon": "i686",
+    "i686": "i586",
+    "geode": "i586",
+    "i586": "i486",
+    "i486": "i386",
+    "i386": "noarch",
+
+    # amd64
+    "x86_64": "athlon",
+    "amd64": "x86_64",
+    "ia32e": "x86_64",
+
+    # ppc64le
+    "ppc64le": "noarch",
+
+    # ppc
+    "ppc64p7": "ppc64",
+    "ppc64pseries": "ppc64",
+    "ppc64iseries": "ppc64",
+    "ppc64": "ppc",
+    "ppc": "noarch",
+
+    # s390{,x}
+    "s390x": "s390",
+    "s390": "noarch",
+
+    # sparc
+    "sparc64v": "sparcv9v",
+    "sparc64": "sparcv9",
+    "sparcv9v": "sparcv9",
+    "sparcv9": "sparcv8",
+    "sparcv8": "sparc",
+    "sparc": "noarch",
+
+    # alpha
+    "alphaev7": "alphaev68",
+    "alphaev68": "alphaev67",
+    "alphaev67": "alphaev6",
+    "alphaev6": "alphapca56",
+    "alphapca56": "alphaev56",
+    "alphaev56": "alphaev5",
+    "alphaev5": "alphaev45",
+    "alphaev45": "alphaev4",
+    "alphaev4": "alpha",
+    "alpha": "noarch",
+
+    # arm
+    "armv7l": "armv6l",
+    "armv6l": "armv5tejl",
+    "armv5tejl": "armv5tel",
+    "armv5tel": "noarch",
+
+    # arm hardware floating point
+    "armv7hnl": "armv7hl",
+    "armv7hl": "armv6hl",
+    "armv6hl": "noarch",
+
+    # arm64
+    "arm64": "noarch",
+
+    # aarch64
+    "aarch64": "noarch",
+
+    # super-h
+    "sh4a": "sh4",
+    "sh4": "noarch",
+    "sh3": "noarch",
+
+    # itanium
+    "ia64": "noarch",
+}
+
+#  Will contain information parsed from /proc/self/auxv via _parse_auxv().
+# Should move into rpm really.
+_aux_vector = {
+    "platform": "",
+    "hwcap": 0,
+}
+
+
+def isMultiLibArch(arch=None):  # pragma: no cover
+    """returns true if arch is a multilib arch, false if not"""
+    if arch is None:
+        arch = canonArch
+
+    if arch not in arches:  # or we could check if it is noarch
+        return 0
+
+    if arch in multilibArches:
+        return 1
+
+    if arches[arch] in multilibArches:
+        return 1
+
+    return 0
+
+
+def getArchList(thisarch=None):  # pragma: no cover
+    # this returns a list of archs that are compatible with arch given
+    if not thisarch:
+        thisarch = canonArch
+
+    archlist = [thisarch]
+    while thisarch in arches:
+        thisarch = arches[thisarch]
+        archlist.append(thisarch)
+
+    # hack hack hack
+    # sparc64v is also sparc64 compat
+    if archlist[0] == "sparc64v":
+        archlist.insert(1, "sparc64")
+
+    # if we're a weirdo arch - add noarch on there.
+    if len(archlist) == 1 and archlist[0] == thisarch:
+        archlist.append('noarch')
+    return archlist
+
+
+def _try_read_cpuinfo():  # pragma: no cover
+    """ Try to read /proc/cpuinfo ... if we can't ignore errors (ie. proc not
+        mounted). """
+    try:
+        return open("/proc/cpuinfo", "r")
+    except:
+        return []
+
+
+def _parse_auxv():  # pragma: no cover
+    """ Read /proc/self/auxv and parse it into global dict for easier access
+        later on, very similar to what rpm does. """
+    # In case we can't open and read /proc/self/auxv, just return
+    try:
+        data = open("/proc/self/auxv", "rb").read()
+    except:
+        return
+
+    # Define values from /usr/include/elf.h
+    AT_PLATFORM = 15
+    AT_HWCAP = 16
+    fmtlen = struct.calcsize("LL")
+    offset = 0
+    platform = ctypes.c_char_p()
+
+    # Parse the data and fill in _aux_vector dict
+    while offset <= len(data) - fmtlen:
+        at_type, at_val = struct.unpack_from("LL", data, offset)
+        if at_type == AT_PLATFORM:
+            platform.value = at_val
+            _aux_vector["platform"] = platform.value
+        if at_type == AT_HWCAP:
+            _aux_vector["hwcap"] = at_val
+        offset = offset + fmtlen
+
+
+def getCanonX86Arch(arch):  # pragma: no cover
+    if arch == "i586":
+        for line in _try_read_cpuinfo():
+            if line.startswith("model name"):
+                if line.find("Geode(TM)") != -1:
+                    return "geode"
+                break
+        return arch
+    # only athlon vs i686 isn't handled with uname currently
+    if arch != "i686":
+        return arch
+
+    # if we're i686 and AuthenticAMD, then we should be an athlon
+    for line in _try_read_cpuinfo():
+        if line.startswith("vendor") and line.find("AuthenticAMD") != -1:
+            return "athlon"
+        # i686 doesn't guarantee cmov, but we depend on it
+        elif line.startswith("flags"):
+            if line.find("cmov") == -1:
+                return "i586"
+            break
+
+    return arch
+
+
+def getCanonARMArch(arch):  # pragma: no cover
+    # the %{_target_arch} macro in rpm will let us know the abi we are using
+    target = rpm.expandMacro('%{_target_cpu}')
+    if target.startswith('armv6h'):
+        return target
+    if target.startswith('armv7h'):
+        return target
+    return arch
+
+
+def getCanonPPCArch(arch):  # pragma: no cover
+    # FIXME: should I do better handling for mac, etc?
+    if arch != "ppc64":
+        return arch
+
+    machine = None
+    for line in _try_read_cpuinfo():
+        if line.find("machine") != -1:
+            machine = line.split(':')[1]
+            break
+
+    platform = _aux_vector["platform"]
+    if machine is None and not platform:
+        return arch
+
+    try:
+        if platform.startswith("power") and int(platform[5:].rstrip('+')) >= 7:
+            return "ppc64p7"
+    except:
+        pass
+
+    if machine is None:
+        return arch
+
+    if machine.find("CHRP IBM") != -1:
+        return "ppc64pseries"
+    if machine.find("iSeries") != -1:
+        return "ppc64iseries"
+    return arch
+
+
+def getCanonSPARCArch(arch):  # pragma: no cover
+    # Deal with sun4v, sun4u, sun4m cases
+    SPARCtype = None
+    for line in _try_read_cpuinfo():
+        if line.startswith("type"):
+            SPARCtype = line.split(':')[1]
+            break
+    if SPARCtype is None:
+        return arch
+
+    if SPARCtype.find("sun4v") != -1:
+        if arch.startswith("sparc64"):
+            return "sparc64v"
+        else:
+            return "sparcv9v"
+    if SPARCtype.find("sun4u") != -1:
+        if arch.startswith("sparc64"):
+            return "sparc64"
+        else:
+            return "sparcv9"
+    if SPARCtype.find("sun4m") != -1:
+        return "sparcv8"
+    return arch
+
+
+def getCanonX86_64Arch(arch):  # pragma: no cover
+    if arch != "x86_64":
+        return arch
+
+    vendor = None
+    for line in _try_read_cpuinfo():
+        if line.startswith("vendor_id"):
+            vendor = line.split(':')[1]
+            break
+    if vendor is None:
+        return arch
+
+    if vendor.find("Authentic AMD") != -1 or vendor.find("AuthenticAMD") != -1:
+        return "amd64"
+    if vendor.find("GenuineIntel") != -1:
+        return "ia32e"
+    return arch
+
+
+def getCanonArch(skipRpmPlatform=0):  # pragma: no cover
+    if not skipRpmPlatform and os.access("/etc/rpm/platform", os.R_OK):
+        try:
+            f = open("/etc/rpm/platform", "r")
+            line = f.readline()
+            f.close()
+            (arch, vendor, opersys) = line.split("-", 2)
+            return arch
+        except:
+            pass
+
+    arch = os.uname()[4]
+
+    _parse_auxv()
+
+    if (len(arch) == 4 and arch[0] == "i" and arch[2:4] == "86"):
+        return getCanonX86Arch(arch)
+
+    if arch.startswith("arm"):
+        return getCanonARMArch(arch)
+    if arch.startswith("ppc"):
+        return getCanonPPCArch(arch)
+    if arch.startswith("sparc"):
+        return getCanonSPARCArch(arch)
+    if arch == "x86_64":
+        return getCanonX86_64Arch(arch)
+
+    return arch
+
+canonArch = getCanonArch()
+
+
+# this gets you the "compat" arch of a biarch pair
+def getMultiArchInfo(arch=canonArch):  # pragma: no cover
+    if arch in multilibArches:
+        return multilibArches[arch]
+    if arch in arches and arches[arch] != "noarch":
+        return getMultiArchInfo(arch=arches[arch])
+    return None
+
+
+def getBaseArch(myarch=None):  # pragma: no cover
+    """returns 'base' arch for myarch, if specified, or canonArch if not.
+       base arch is the arch before noarch in the arches dict if myarch is not
+       a key in the multilibArches."""
+
+    if not myarch:
+        myarch = canonArch
+
+    if myarch not in arches:  # this is dumb, but <shrug>
+        return myarch
+
+    if myarch.startswith("sparc64"):
+        return "sparc"
+    elif myarch == "ppc64le":
+        return "ppc64le"
+    elif myarch.startswith("ppc64") and not _ppc64_native_is_best:
+        return "ppc"
+    elif myarch.startswith("arm64"):
+        return "arm64"
+    elif myarch.startswith("armv6h"):
+        return "armhfp"
+    elif myarch.startswith("armv7h"):
+        return "armhfp"
+    elif myarch.startswith("arm"):
+        return "arm"
+
+    if isMultiLibArch(arch=myarch):
+        if myarch in multilibArches:
+            return myarch
+        else:
+            return arches[myarch]
+
+    if myarch in arches:
+        basearch = myarch
+        value = arches[basearch]
+        while value != 'noarch':
+            basearch = value
+            value = arches[basearch]
+
+        return basearch

pungi/config.py

@@ -17,11 +17,10 @@
 import os
 import sys
 import time
-import yum
 
 from ConfigParser import SafeConfigParser
 
-
+from .arch_utils import getBaseArch
 # In development, `here` will point to the bin/ directory with scripts.
 here = sys.path[0]
 MULTILIBCONF = (os.path.join(os.path.dirname(__file__), '..', 'share', 'multilib')
@@ -47,7 +46,7 @@ class Config(SafeConfigParser):
         self.set('pungi', 'product_path', 'Packages')
         self.set('pungi', 'cachedir', '/var/cache/pungi')
         self.set('pungi', 'compress_type', 'xz')
-        self.set('pungi', 'arch', yum.rpmUtils.arch.getBaseArch())
+        self.set('pungi', 'arch', getBaseArch())
         self.set('pungi', 'family', 'Fedora')
         self.set('pungi', 'iso_basename', 'Fedora')
         self.set('pungi', 'version', time.strftime('%Y%m%d', time.localtime()))

pungi/ostree/utils.py

@@ -18,8 +18,8 @@ import datetime
 import json
 import logging
 import os
-import rpmUtils.arch
 
+from pungi.arch_utils import getBaseArch
 from pungi.util import makedirs
 
 
@@ -43,9 +43,9 @@ def get_ref_from_treefile(treefile, arch=None, logger=None):
             try:
                 parsed = json.loads(f.read())
                 if arch is None:
-                    basearch = rpmUtils.arch.getBaseArch()
+                    basearch = getBaseArch()
                 else:
-                    basearch = rpmUtils.arch.getBaseArch(arch)
+                    basearch = getBaseArch(arch)
                 ref = parsed['ref'].replace('${basearch}', basearch)
             except Exception as e:
                 logger.error('Unable to get ref from treefile: %s' % e)

pungi/wrappers/kojiwrapper.py

@@ -22,11 +22,11 @@ import threading
 import contextlib
 
 import koji
-import rpmUtils.arch
 from kobo.shortcuts import run
 from ConfigParser import ConfigParser
 
 from .. import util
+from ..arch_utils import getBaseArch
 
 
 class KojiWrapper(object):
@@ -100,7 +100,7 @@ class KojiWrapper(object):
         cmd.append(target)
 
         # i686 -> i386 etc.
-        arch = rpmUtils.arch.getBaseArch(arch)
+        arch = getBaseArch(arch)
         cmd.append(arch)
 
         if isinstance(command, list):
@@ -280,7 +280,7 @@ class KojiWrapper(object):
         cmd.append(target)
 
         # i686 -> i386 etc.
-        arch = rpmUtils.arch.getBaseArch(arch)
+        arch = getBaseArch(arch)
         cmd.append(arch)
 
         cmd.append(ks_file)

tests/test_arch.py

@@ -56,7 +56,7 @@ class MockArchModule(object):
         return ARCHES[yum_arch]
 
 
-@mock.patch('rpmUtils.arch', MockArchModule)
+@mock.patch('pungi.arch_utils', MockArchModule)
 class TestArch(unittest.TestCase):
 
     def test_i386(self):