Convert pylorax to python3

Fix up 2to3 complaints. I've decided to do with wrapping list
comprehension inside list() to get the generators to run in several
places instead of list(map( or list(filter( which seem less readable to
me.

src/pylorax/__init__.py

@@ -1,7 +1,7 @@
 #
 # __init__.py
 #
-# Copyright (C) 2010-2014  Red Hat, Inc.
+# Copyright (C) 2010-2015  Red Hat, Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -27,7 +27,7 @@ logger.addHandler(logging.NullHandler())
 
 import sys
 import os
-import ConfigParser
+import configparser
 import tempfile
 import locale
 from subprocess import CalledProcessError
@@ -39,7 +39,6 @@ import pylorax.output as output
 import dnf
 
 from pylorax.sysutils import joinpaths, remove, linktree
-from rpmUtils.arch import getBaseArch
 
 from pylorax.treebuilder import RuntimeBuilder, TreeBuilder
 from pylorax.buildstamp import BuildStamp
@@ -60,7 +59,7 @@ class ArchData(DataHolder):
     def __init__(self, buildarch):
         super(ArchData, self).__init__()
         self.buildarch = buildarch
-        self.basearch = getBaseArch(buildarch)
+        self.basearch = dnf.arch.basearch(buildarch)
         self.libdir = "lib64" if self.basearch in self.lib64_arches else "lib"
         self.bcj = self.bcj_arch.get(self.basearch)
 
@@ -81,7 +80,7 @@ class Lorax(BaseLoraxClass):
         locale.setlocale(locale.LC_ALL, 'C')
 
     def configure(self, conf_file="/etc/lorax/lorax.conf"):
-        self.conf = ConfigParser.SafeConfigParser()
+        self.conf = configparser.SafeConfigParser()
 
         # set defaults
         self.conf.add_section("lorax")
@@ -133,7 +132,7 @@ class Lorax(BaseLoraxClass):
 
         # remove some environmental variables that can cause problems with package scripts
         env_remove = ('DISPLAY', 'DBUS_SESSION_BUS_ADDRESS')
-        map(os.environ.pop, (k for k in env_remove if k in os.environ))
+        list(os.environ.pop(k) for k in env_remove if k in os.environ)
 
         self._configured = True
 

src/pylorax/base.py

@@ -1,7 +1,7 @@
 #
 # base.py
 #
-# Copyright (C) 2009-2014 Red Hat, Inc.
+# Copyright (C) 2009-2015 Red Hat, Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -25,10 +25,7 @@ import sys
 import pylorax.output as output
 
 
-class BaseLoraxClass(object):
+class BaseLoraxClass(object, metaclass=ABCMeta):
-
-    __metaclass__ = ABCMeta
-
     @abstractmethod
     def __init__(self):
         self.output = output.LoraxOutput()

src/pylorax/buildstamp.py

@@ -1,7 +1,7 @@
 #
 # buildstamp.py
 #
-# Copyright (C) 2010-2014 Red Hat, Inc.
+# Copyright (C) 2010-2015 Red Hat, Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

src/pylorax/decorators.py

@@ -1,7 +1,7 @@
 #
 # decorators.py
 #
-# Copyright (C) 2009-2014 Red Hat, Inc.
+# Copyright (C) 2009-2015 Red Hat, Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

src/pylorax/discinfo.py

@@ -1,7 +1,7 @@
 #
 # discinfo.py
 #
-# Copyright (C) 2010-2014  Red Hat, Inc.
+# Copyright (C) 2010-2015  Red Hat, Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

src/pylorax/dnfhelper.py

@@ -1,7 +1,7 @@
 #
 # dnfhelper.py
 #
-# Copyright (C) 2010-2014 Red Hat, Inc.
+# Copyright (C) 2010-2015 Red Hat, Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

src/pylorax/executils.py

@@ -1,7 +1,7 @@
 #
 # executil.py - subprocess execution utility functions
 #
-# Copyright (C) 1999-2014
+# Copyright (C) 1999-2015
 # Red Hat, Inc.  All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify
@@ -21,6 +21,7 @@
 #
 
 import os, sys
+import io
 import subprocess
 import threading
 from time import sleep
@@ -98,7 +99,7 @@ def execWithRedirect(command, argv, stdin = None, stdout = None,
             stdin = sys.stdin.fileno()
     elif isinstance(stdin, int):
         pass
-    elif stdin is None or not isinstance(stdin, file):
+    elif stdin is None or not isinstance(stdin, io.IOBase):
         stdin = sys.stdin.fileno()
 
     if isinstance(stdout, str):
@@ -106,7 +107,7 @@ def execWithRedirect(command, argv, stdin = None, stdout = None,
         stdoutclose = lambda : os.close(stdout)
     elif isinstance(stdout, int):
         pass
-    elif stdout is None or not isinstance(stdout, file):
+    elif stdout is None or not isinstance(stdout, io.IOBase):
         stdout = sys.stdout.fileno()
 
     if isinstance(stderr, str):
@@ -114,7 +115,7 @@ def execWithRedirect(command, argv, stdin = None, stdout = None,
         stderrclose = lambda : os.close(stderr)
     elif isinstance(stderr, int):
         pass
-    elif stderr is None or not isinstance(stderr, file):
+    elif stderr is None or not isinstance(stderr, io.IOBase):
         stderr = sys.stderr.fileno()
 
     program_log.info("Running... %s", " ".join([command] + argv))
@@ -220,7 +221,7 @@ def execWithCapture(command, argv, stdin = None, stderr = None, root=None,
             stdin = sys.stdin.fileno()
     elif isinstance(stdin, int):
         pass
-    elif stdin is None or not isinstance(stdin, file):
+    elif stdin is None or not isinstance(stdin, io.IOBase):
         stdin = sys.stdin.fileno()
 
     if isinstance(stderr, str):
@@ -228,7 +229,7 @@ def execWithCapture(command, argv, stdin = None, stderr = None, root=None,
         stderrclose = lambda : os.close(stderr)
     elif isinstance(stderr, int):
         pass
-    elif stderr is None or not isinstance(stderr, file):
+    elif stderr is None or not isinstance(stderr, io.IOBase):
         stderr = sys.stderr.fileno()
 
     program_log.info("Running... %s", " ".join([command] + argv))
@@ -253,10 +254,10 @@ def execWithCapture(command, argv, stdin = None, stderr = None, root=None,
         while True:
             (outStr, errStr) = proc.communicate()
             if outStr:
-                map(program_log.info, outStr.splitlines())
+                list(program_log.info(line) for line in outStr.splitlines())
                 rc += outStr
             if errStr:
-                map(program_log.error, errStr.splitlines())
+                list(program_log.error(line) for line in errStr.splitlines())
                 os.write(stderr, errStr)
 
             if proc.returncode is not None:
@@ -291,7 +292,7 @@ def execWithCallback(command, argv, stdin = None, stdout = None,
             stdin = sys.stdin.fileno()
     elif isinstance(stdin, int):
         pass
-    elif stdin is None or not isinstance(stdin, file):
+    elif stdin is None or not isinstance(stdin, io.IOBase):
         stdin = sys.stdin.fileno()
 
     if isinstance(stdout, str):
@@ -299,7 +300,7 @@ def execWithCallback(command, argv, stdin = None, stdout = None,
         stdoutclose = lambda : os.close(stdout)
     elif isinstance(stdout, int):
         pass
-    elif stdout is None or not isinstance(stdout, file):
+    elif stdout is None or not isinstance(stdout, io.IOBase):
         stdout = sys.stdout.fileno()
 
     if isinstance(stderr, str):
@@ -307,7 +308,7 @@ def execWithCallback(command, argv, stdin = None, stdout = None,
         stderrclose = lambda : os.close(stderr)
     elif isinstance(stderr, int):
         pass
-    elif stderr is None or not isinstance(stderr, file):
+    elif stderr is None or not isinstance(stderr, io.IOBase):
         stderr = sys.stderr.fileno()
 
     program_log.info("Running... %s", " ".join([command] + argv))
@@ -337,7 +338,7 @@ def execWithCallback(command, argv, stdin = None, stdout = None,
             s = os.read(p[0], 1)
         except OSError as e:
             if e.errno != 4:
-                map(program_log.info, log_output.splitlines())
+                list(program_log.info(line) for line in log_output.splitlines())
                 raise IOError(e.args)
 
         if echo:
@@ -359,7 +360,7 @@ def execWithCallback(command, argv, stdin = None, stdout = None,
         if len(s) < 1:
             break
 
-    map(program_log.info, log_output.splitlines())
+    list(program_log.info(line) for line in log_output.splitlines())
 
     log_errors = ''
     while 1:
@@ -367,7 +368,7 @@ def execWithCallback(command, argv, stdin = None, stdout = None,
             err = os.read(p_stderr[0], 128)
         except OSError as e:
             if e.errno != 4:
-                map(program_log.error, log_errors.splitlines())
+                list(program_log.error(line) for line in log_errors.splitlines())
                 raise IOError(e.args)
             break
         log_errors += err
@@ -375,7 +376,7 @@ def execWithCallback(command, argv, stdin = None, stdout = None,
             break
 
     os.write(stderr, log_errors)
-    map(program_log.error, log_errors.splitlines())
+    list(program_log.error(line) for line in log_errors.splitlines())
     os.close(p[0])
     os.close(p_stderr[0])
 

src/pylorax/imgutils.py

@@ -1,6 +1,6 @@
 # imgutils.py - utility functions/classes for building disk images
 #
-# Copyright (C) 2011-2014 Red Hat, Inc.
+# Copyright (C) 2011-2015 Red Hat, Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -71,7 +71,7 @@ def compress(command, rootdir, outfile, compression="xz", compressargs=None):
     except OSError as e:
         logger.error(e)
         # Kill off any hanging processes
-        map(lambda p: p.kill(), (p for p in (find, archive, comp) if p))
+        list(p.kill() for p in (find, archive, comp) if p)
         return 1
 
 def mkcpio(rootdir, outfile, compression="xz", compressargs=None):
@@ -259,7 +259,7 @@ def estimate_size(rootdir, graft=None, fstype=None, blocksize=4096, overhead=128
         blocksize = 2048
         getsize = lambda f: os.stat(f).st_size # no symlinks, count as copies
     total = overhead*blocksize
-    dirlist = graft.values()
+    dirlist = list(graft.values())
     if rootdir:
         dirlist.append(rootdir)
     for root in dirlist:

src/pylorax/ltmpl.py

@@ -1,7 +1,7 @@
 #
 # ltmpl.py
 #
-# Copyright (C) 2009-2014  Red Hat, Inc.
+# Copyright (C) 2009-2015  Red Hat, Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -40,7 +40,8 @@ import sys, traceback
 import struct
 import dnf
 import multiprocessing
-import Queue
+import queue
+import collections
 
 class LoraxTemplate(object):
     def __init__(self, directories=None):
@@ -62,17 +63,17 @@ class LoraxTemplate(object):
 
         # split, strip and remove empty lines
         lines = textbuf.splitlines()
-        lines = map(lambda line: line.strip(), lines)
+        lines = [line.strip() for line in lines]
-        lines = filter(lambda line: line, lines)
+        lines = [line for line in lines if line]
 
         # remove comments
-        lines = filter(lambda line: not line.startswith("#"), lines)
+        lines = [line for line in lines if not line.startswith("#")]
 
         # mako template now returns unicode strings
-        lines = map(lambda line: line.encode("utf8"), lines)
+        lines = [line.encode("utf8") for line in lines]
 
         # split with shlex and perform brace expansion
-        lines = map(split_and_expand, lines)
+        lines = [split_and_expand(line) for line in lines]
 
         return lines
 
@@ -184,7 +185,7 @@ class LoraxTemplateRunner(object):
         return sum(os.path.getsize(self._out(f)) for f in files if os.path.isfile(self._out(f)))
 
     def run(self, templatefile, **variables):
-        for k,v in self.defaults.items() + self.builtins.items():
+        for k,v in list(self.defaults.items()) + list(self.builtins.items()):
             variables.setdefault(k,v)
         logger.debug("executing %s with variables=%s", templatefile, variables)
         self.templatefile = templatefile
@@ -207,7 +208,7 @@ class LoraxTemplateRunner(object):
             try:
                 # grab the method named in cmd and pass it the given arguments
                 f = getattr(self, cmd, None)
-                if cmd[0] == '_' or cmd == 'run' or not callable(f):
+                if cmd[0] == '_' or cmd == 'run' or not isinstance(f, collections.Callable):
                     raise ValueError("unknown command %s" % cmd)
                 f(*args)
             except Exception: # pylint: disable=broad-except
@@ -506,18 +507,18 @@ class LoraxTemplateRunner(object):
             else:
                 logger.debug("removepkg %s: no files to remove!", p)
 
-    def get_token_checked(self, process, queue):
+    def get_token_checked(self, process, token_queue):
         """Try to get token from queue checking that process is still alive"""
 
         try:
             # wait at most a minute for the token
-            (token, msg) = queue.get(timeout=60)
+            (token, msg) = token_queue.get(timeout=60)
-        except Queue.Empty:
+        except queue.Empty:
             if process.is_alive():
                 try:
                     # process still alive, give it 2 minutes more
-                    (token, msg) = queue.get(timeout=120)
+                    (token, msg) = token_queue.get(timeout=120)
-                except Queue.Empty:
+                except queue.Empty:
                     # waited for 3 minutes and got nothing
                     raise Exception("The transaction process got stuck somewhere (no message from it in 3 minutes)")
             else:
@@ -532,13 +533,13 @@ class LoraxTemplateRunner(object):
           commands.
         '''
 
-        def do_transaction(base, queue):
+        def do_transaction(base, token_queue):
             try:
-                display = LoraxRpmCallback(queue)
+                display = LoraxRpmCallback(token_queue)
                 base.do_transaction(display=display)
             except BaseException as e:
                 logger.error("The transaction process has ended abruptly: %s", e)
-                queue.put(('quit', str(e)))
+                token_queue.put(('quit', str(e)))
 
         try:
             logger.info("Checking dependencies")
@@ -560,17 +561,17 @@ class LoraxTemplateRunner(object):
             raise
 
         logger.info("Preparing transaction from installation source")
-        queue = multiprocessing.Queue()
+        token_queue = multiprocessing.Queue()
         msgout = output.LoraxOutput()
-        process = multiprocessing.Process(target=do_transaction, args=(self.dbo, queue))
+        process = multiprocessing.Process(target=do_transaction, args=(self.dbo, token_queue))
         process.start()
-        (token, msg) = self.get_token_checked(process, queue)
+        (token, msg) = self.get_token_checked(process, token_queue)
 
         while token not in ('post', 'quit'):
             if token == 'install':
                 logging.info("%s", msg)
                 msgout.writeline(msg)
-            (token, msg) = self.get_token_checked(process, queue)
+            (token, msg) = self.get_token_checked(process, token_queue)
 
         if token == 'quit':
             logger.error("Transaction failed.")
@@ -608,7 +609,7 @@ class LoraxTemplateRunner(object):
         matches = set()
         for g in globs:
             globs_re = re.compile(fnmatch.translate(g))
-            m = filter(globs_re.match, filelist)
+            m = [f for f in filelist if globs_re.match(f)]
             if m:
                 matches.update(m)
             else:
@@ -670,7 +671,7 @@ class LoraxTemplateRunner(object):
         matches = set()
         for g in keepglobs:
             globs_re = re.compile(fnmatch.translate("*"+g+"*"))
-            m = filter(globs_re.match, filelist)
+            m = [f for f in filelist if globs_re.match(f)]
             if m:
                 matches.update(m)
             else:
@@ -679,7 +680,7 @@ class LoraxTemplateRunner(object):
 
         if remove_files:
             logger.debug("removekmod: removing %d files", len(remove_files))
-            map(remove, remove_files)
+            list(remove(f) for f in remove_files)
         else:
             logger.debug("removekmod %s: no files to remove!", cmd)
 

src/pylorax/output.py

@@ -1,7 +1,7 @@
 #
 # output.py
 #
-# Copyright (C) 2009-2014 Red Hat, Inc.
+# Copyright (C) 2009-2015 Red Hat, Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

src/pylorax/sysutils.py

@@ -1,7 +1,7 @@
 #
 # sysutils.py
 #
-# Copyright (C) 2009-2014 Red Hat, Inc.
+# Copyright (C) 2009-2015 Red Hat, Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -43,8 +43,8 @@ def joinpaths(*args, **kwargs):
 
 
 def touch(fname):
-    with open(fname, "w") as _:
+    # python closes the file when it goes out of scope
-        pass
+    open(fname, "w").write("")
 
 
 def replace(fname, find, sub):

src/pylorax/treebuilder.py

@@ -1,6 +1,6 @@
 # treebuilder.py - handle arch-specific tree building stuff using templates
 #
-# Copyright (C) 2011-2014 Red Hat, Inc.
+# Copyright (C) 2011-2015 Red Hat, Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -324,7 +324,7 @@ def findkernels(root="/", kdir="boot"):
 udev_blacklist=' !"$%&\'()*,/;<>?[\\]^`{|}~' # ASCII printable, minus whitelist
 udev_blacklist += ''.join(chr(i) for i in range(32)) # ASCII non-printable
 def udev_escape(label):
-    out = u''
+    out = ''
     for ch in label.decode('utf8'):
-        out += ch if ch not in udev_blacklist else u'\\x%02x' % ord(ch)
+        out += ch if ch not in udev_blacklist else '\\x%02x' % ord(ch)
     return out.encode('utf8')

src/pylorax/treeinfo.py

@@ -1,7 +1,7 @@
 #
 # treeinfo.py
 #
-# Copyright (C) 2010-2014 Red Hat, Inc.
+# Copyright (C) 2010-2015 Red Hat, Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -22,7 +22,7 @@
 import logging
 logger = logging.getLogger("pylorax.treeinfo")
 
-import ConfigParser
+import configparser
 import time
 
 
@@ -31,7 +31,7 @@ class TreeInfo(object):
     def __init__(self, product, version, variant, basearch,
                  packagedir=""):
 
-        self.c = ConfigParser.ConfigParser()
+        self.c = configparser.ConfigParser()
 
         section = "general"
         data = {"timestamp": time.time(),
@@ -43,13 +43,13 @@ class TreeInfo(object):
                 "packagedir": packagedir}
 
         self.c.add_section(section)
-        map(lambda (key, value): self.c.set(section, key, value), data.items())
+        list(self.c.set(section, key, value) for key, value in data.items())
 
     def add_section(self, section, data):
         if not self.c.has_section(section):
             self.c.add_section(section)
 
-        map(lambda (key, value): self.c.set(section, key, value), data.items())
+        list(self.c.set(section, key, value) for key, value in data.items())
 
     def write(self, outfile):
         logger.info("writing .treeinfo file")

tests/pylint/pylint-one.sh

@@ -10,7 +10,7 @@ fi
 
 file_suffix="$(eval echo \$$#|sed s?/?_?g)"
 
-pylint_output="$(pylint \
+pylint_output="$(python3-pylint \
     --msg-template='{path}:{line}: {msg_id} {symbol} {msg}' \
     -r n --disable=C,R --rcfile=/dev/null \
     --dummy-variables-rgx=_ \