almalinux
/
lorax
14
0
Fork 0
Code Issues Pull Requests Releases Activity

Convert pylorax to python3 

Fix up 2to3 complaints. I've decided to do with wrapping list
comprehension inside list() to get the generators to run in several
places instead of list(map( or list(filter( which seem less readable to
me.
This commit is contained in:
Brian C. Lane 2015-05-06 10:38:57 -07:00
parent a6e469e0e6
commit de0e662f51
14 changed files with 68 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/pylorax/__init__.py
View File

@ -1,7 +1,7 @@
#
# __init__.py
#
# Copyright (C) 2010-2014 Red Hat, Inc.
# Copyright (C) 2010-2015 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -27,7 +27,7 @@ logger.addHandler(logging.NullHandler())
import sys
import os
import ConfigParser
import configparser
import tempfile
import locale
from subprocess import CalledProcessError
@ -39,7 +39,6 @@ import pylorax.output as output
import dnf
from pylorax.sysutils import joinpaths, remove, linktree
from rpmUtils.arch import getBaseArch
from pylorax.treebuilder import RuntimeBuilder, TreeBuilder
from pylorax.buildstamp import BuildStamp
@ -60,7 +59,7 @@ class ArchData(DataHolder):
def __init__(self, buildarch):
super(ArchData, self).__init__()
self.buildarch = buildarch
self.basearch = getBaseArch(buildarch)
self.basearch = dnf.arch.basearch(buildarch)
self.libdir = "lib64" if self.basearch in self.lib64_arches else "lib"
self.bcj = self.bcj_arch.get(self.basearch)
@ -81,7 +80,7 @@ class Lorax(BaseLoraxClass):
locale.setlocale(locale.LC_ALL, 'C')
def configure(self, conf_file="/etc/lorax/lorax.conf"):
self.conf = ConfigParser.SafeConfigParser()
self.conf = configparser.SafeConfigParser()
# set defaults
self.conf.add_section("lorax")
@ -133,7 +132,7 @@ class Lorax(BaseLoraxClass):
# remove some environmental variables that can cause problems with package scripts
env_remove = ('DISPLAY', 'DBUS_SESSION_BUS_ADDRESS')
map(os.environ.pop, (k for k in env_remove if k in os.environ))
list(os.environ.pop(k) for k in env_remove if k in os.environ)
self._configured = True

7
src/pylorax/base.py
View File

@ -1,7 +1,7 @@
#
# base.py
#
# Copyright (C) 2009-2014 Red Hat, Inc.
# Copyright (C) 2009-2015 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -25,10 +25,7 @@ import sys
import pylorax.output as output
class BaseLoraxClass(object):
__metaclass__ = ABCMeta
class BaseLoraxClass(object, metaclass=ABCMeta):
@abstractmethod
def __init__(self):
self.output = output.LoraxOutput()

2
src/pylorax/buildstamp.py
View File

@ -1,7 +1,7 @@
#
# buildstamp.py
#
# Copyright (C) 2010-2014 Red Hat, Inc.
# Copyright (C) 2010-2015 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by

2
src/pylorax/decorators.py
View File

@ -1,7 +1,7 @@
#
# decorators.py
#
# Copyright (C) 2009-2014 Red Hat, Inc.
# Copyright (C) 2009-2015 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by

2
src/pylorax/discinfo.py
View File

@ -1,7 +1,7 @@
#
# discinfo.py
#
# Copyright (C) 2010-2014 Red Hat, Inc.
# Copyright (C) 2010-2015 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by

2
src/pylorax/dnfhelper.py
View File

@ -1,7 +1,7 @@
#
# dnfhelper.py
#
# Copyright (C) 2010-2014 Red Hat, Inc.
# Copyright (C) 2010-2015 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by

31
src/pylorax/executils.py
View File

@ -1,7 +1,7 @@
#
# executil.py - subprocess execution utility functions
#
# Copyright (C) 1999-2014
# Copyright (C) 1999-2015
# Red Hat, Inc. All rights reserved.
#
# This program is free software; you can redistribute it and/or modify
@ -21,6 +21,7 @@
#
import os, sys
import io
import subprocess
import threading
from time import sleep
@ -98,7 +99,7 @@ def execWithRedirect(command, argv, stdin = None, stdout = None,
stdin = sys.stdin.fileno()
elif isinstance(stdin, int):
pass
elif stdin is None or not isinstance(stdin, file):
elif stdin is None or not isinstance(stdin, io.IOBase):
stdin = sys.stdin.fileno()
if isinstance(stdout, str):
@ -106,7 +107,7 @@ def execWithRedirect(command, argv, stdin = None, stdout = None,
stdoutclose = lambda : os.close(stdout)
elif isinstance(stdout, int):
pass
elif stdout is None or not isinstance(stdout, file):
elif stdout is None or not isinstance(stdout, io.IOBase):
stdout = sys.stdout.fileno()
if isinstance(stderr, str):
@ -114,7 +115,7 @@ def execWithRedirect(command, argv, stdin = None, stdout = None,
stderrclose = lambda : os.close(stderr)
elif isinstance(stderr, int):
pass
elif stderr is None or not isinstance(stderr, file):
elif stderr is None or not isinstance(stderr, io.IOBase):
stderr = sys.stderr.fileno()
program_log.info("Running... %s", " ".join([command] + argv))
@ -220,7 +221,7 @@ def execWithCapture(command, argv, stdin = None, stderr = None, root=None,
stdin = sys.stdin.fileno()
elif isinstance(stdin, int):
pass
elif stdin is None or not isinstance(stdin, file):
elif stdin is None or not isinstance(stdin, io.IOBase):
stdin = sys.stdin.fileno()
if isinstance(stderr, str):
@ -228,7 +229,7 @@ def execWithCapture(command, argv, stdin = None, stderr = None, root=None,
stderrclose = lambda : os.close(stderr)
elif isinstance(stderr, int):
pass
elif stderr is None or not isinstance(stderr, file):
elif stderr is None or not isinstance(stderr, io.IOBase):
stderr = sys.stderr.fileno()
program_log.info("Running... %s", " ".join([command] + argv))
@ -253,10 +254,10 @@ def execWithCapture(command, argv, stdin = None, stderr = None, root=None,
while True:
(outStr, errStr) = proc.communicate()
if outStr:
map(program_log.info, outStr.splitlines())
list(program_log.info(line) for line in outStr.splitlines())
rc += outStr
if errStr:
map(program_log.error, errStr.splitlines())
list(program_log.error(line) for line in errStr.splitlines())
os.write(stderr, errStr)
if proc.returncode is not None:
@ -291,7 +292,7 @@ def execWithCallback(command, argv, stdin = None, stdout = None,
stdin = sys.stdin.fileno()
elif isinstance(stdin, int):
pass
elif stdin is None or not isinstance(stdin, file):
elif stdin is None or not isinstance(stdin, io.IOBase):
stdin = sys.stdin.fileno()
if isinstance(stdout, str):
@ -299,7 +300,7 @@ def execWithCallback(command, argv, stdin = None, stdout = None,
stdoutclose = lambda : os.close(stdout)
elif isinstance(stdout, int):
pass
elif stdout is None or not isinstance(stdout, file):
elif stdout is None or not isinstance(stdout, io.IOBase):
stdout = sys.stdout.fileno()
if isinstance(stderr, str):
@ -307,7 +308,7 @@ def execWithCallback(command, argv, stdin = None, stdout = None,
stderrclose = lambda : os.close(stderr)
elif isinstance(stderr, int):
pass
elif stderr is None or not isinstance(stderr, file):
elif stderr is None or not isinstance(stderr, io.IOBase):
stderr = sys.stderr.fileno()
program_log.info("Running... %s", " ".join([command] + argv))
@ -337,7 +338,7 @@ def execWithCallback(command, argv, stdin = None, stdout = None,
s = os.read(p[0], 1)
except OSError as e:
if e.errno != 4:
map(program_log.info, log_output.splitlines())
list(program_log.info(line) for line in log_output.splitlines())
raise IOError(e.args)
if echo:
@ -359,7 +360,7 @@ def execWithCallback(command, argv, stdin = None, stdout = None,
if len(s) < 1:
break
map(program_log.info, log_output.splitlines())
list(program_log.info(line) for line in log_output.splitlines())
log_errors = ''
while 1:
@ -367,7 +368,7 @@ def execWithCallback(command, argv, stdin = None, stdout = None,
err = os.read(p_stderr[0], 128)
except OSError as e:
if e.errno != 4:
map(program_log.error, log_errors.splitlines())
list(program_log.error(line) for line in log_errors.splitlines())
raise IOError(e.args)
break
log_errors += err
@ -375,7 +376,7 @@ def execWithCallback(command, argv, stdin = None, stdout = None,
break
os.write(stderr, log_errors)
map(program_log.error, log_errors.splitlines())
list(program_log.error(line) for line in log_errors.splitlines())
os.close(p[0])
os.close(p_stderr[0])

6
src/pylorax/imgutils.py
View File

@ -1,6 +1,6 @@
# imgutils.py - utility functions/classes for building disk images
#
# Copyright (C) 2011-2014 Red Hat, Inc.
# Copyright (C) 2011-2015 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -71,7 +71,7 @@ def compress(command, rootdir, outfile, compression="xz", compressargs=None):
except OSError as e:
logger.error(e)
# Kill off any hanging processes
map(lambda p: p.kill(), (p for p in (find, archive, comp) if p))
list(p.kill() for p in (find, archive, comp) if p)
return 1
def mkcpio(rootdir, outfile, compression="xz", compressargs=None):
@ -259,7 +259,7 @@ def estimate_size(rootdir, graft=None, fstype=None, blocksize=4096, overhead=128
blocksize = 2048
getsize = lambda f: os.stat(f).st_size # no symlinks, count as copies
total = overhead*blocksize
dirlist = graft.values()
dirlist = list(graft.values())
if rootdir:
dirlist.append(rootdir)
for root in dirlist:

49
src/pylorax/ltmpl.py
View File

@ -1,7 +1,7 @@
#
# ltmpl.py
#
# Copyright (C) 2009-2014 Red Hat, Inc.
# Copyright (C) 2009-2015 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -40,7 +40,8 @@ import sys, traceback
import struct
import dnf
import multiprocessing
import Queue
import queue
import collections
class LoraxTemplate(object):
def __init__(self, directories=None):
@ -62,17 +63,17 @@ class LoraxTemplate(object):
# split, strip and remove empty lines
lines = textbuf.splitlines()
lines = map(lambda line: line.strip(), lines)
lines = filter(lambda line: line, lines)
lines = [line.strip() for line in lines]
lines = [line for line in lines if line]
# remove comments
lines = filter(lambda line: not line.startswith("#"), lines)
lines = [line for line in lines if not line.startswith("#")]
# mako template now returns unicode strings
lines = map(lambda line: line.encode("utf8"), lines)
lines = [line.encode("utf8") for line in lines]
# split with shlex and perform brace expansion
lines = map(split_and_expand, lines)
lines = [split_and_expand(line) for line in lines]
return lines
@ -184,7 +185,7 @@ class LoraxTemplateRunner(object):
return sum(os.path.getsize(self._out(f)) for f in files if os.path.isfile(self._out(f)))
def run(self, templatefile, **variables):
for k,v in self.defaults.items() + self.builtins.items():
for k,v in list(self.defaults.items()) + list(self.builtins.items()):
variables.setdefault(k,v)
logger.debug("executing %s with variables=%s", templatefile, variables)
self.templatefile = templatefile
@ -207,7 +208,7 @@ class LoraxTemplateRunner(object):
try:
# grab the method named in cmd and pass it the given arguments
f = getattr(self, cmd, None)
if cmd[0] == '_' or cmd == 'run' or not callable(f):
if cmd[0] == '_' or cmd == 'run' or not isinstance(f, collections.Callable):
raise ValueError("unknown command %s" % cmd)
f(*args)
except Exception: # pylint: disable=broad-except
@ -506,18 +507,18 @@ class LoraxTemplateRunner(object):
else:
logger.debug("removepkg %s: no files to remove!", p)
def get_token_checked(self, process, queue):
def get_token_checked(self, process, token_queue):
"""Try to get token from queue checking that process is still alive"""
try:
# wait at most a minute for the token
(token, msg) = queue.get(timeout=60)
except Queue.Empty:
(token, msg) = token_queue.get(timeout=60)
except queue.Empty:
if process.is_alive():
try:
# process still alive, give it 2 minutes more
(token, msg) = queue.get(timeout=120)
except Queue.Empty:
(token, msg) = token_queue.get(timeout=120)
except queue.Empty:
# waited for 3 minutes and got nothing
raise Exception("The transaction process got stuck somewhere (no message from it in 3 minutes)")
else:
@ -532,13 +533,13 @@ class LoraxTemplateRunner(object):
commands.
'''
def do_transaction(base, queue):
def do_transaction(base, token_queue):
try:
display = LoraxRpmCallback(queue)
display = LoraxRpmCallback(token_queue)
base.do_transaction(display=display)
except BaseException as e:
logger.error("The transaction process has ended abruptly: %s", e)
queue.put(('quit', str(e)))
token_queue.put(('quit', str(e)))
try:
logger.info("Checking dependencies")
@ -560,17 +561,17 @@ class LoraxTemplateRunner(object):
raise
logger.info("Preparing transaction from installation source")
queue = multiprocessing.Queue()
token_queue = multiprocessing.Queue()
msgout = output.LoraxOutput()
process = multiprocessing.Process(target=do_transaction, args=(self.dbo, queue))
process = multiprocessing.Process(target=do_transaction, args=(self.dbo, token_queue))
process.start()
(token, msg) = self.get_token_checked(process, queue)
(token, msg) = self.get_token_checked(process, token_queue)
while token not in ('post', 'quit'):
if token == 'install':
logging.info("%s", msg)
msgout.writeline(msg)
(token, msg) = self.get_token_checked(process, queue)
(token, msg) = self.get_token_checked(process, token_queue)
if token == 'quit':
logger.error("Transaction failed.")
@ -608,7 +609,7 @@ class LoraxTemplateRunner(object):
matches = set()
for g in globs:
globs_re = re.compile(fnmatch.translate(g))
m = filter(globs_re.match, filelist)
m = [f for f in filelist if globs_re.match(f)]
if m:
matches.update(m)
else:
@ -670,7 +671,7 @@ class LoraxTemplateRunner(object):
matches = set()
for g in keepglobs:
globs_re = re.compile(fnmatch.translate("*"+g+"*"))
m = filter(globs_re.match, filelist)
m = [f for f in filelist if globs_re.match(f)]
if m:
matches.update(m)
else:
@ -679,7 +680,7 @@ class LoraxTemplateRunner(object):
if remove_files:
logger.debug("removekmod: removing %d files", len(remove_files))
map(remove, remove_files)
list(remove(f) for f in remove_files)
else:
logger.debug("removekmod %s: no files to remove!", cmd)

2
src/pylorax/output.py
View File

@ -1,7 +1,7 @@
#
# output.py
#
# Copyright (C) 2009-2014 Red Hat, Inc.
# Copyright (C) 2009-2015 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by

6
src/pylorax/sysutils.py
View File

@ -1,7 +1,7 @@
#
# sysutils.py
#
# Copyright (C) 2009-2014 Red Hat, Inc.
# Copyright (C) 2009-2015 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -43,8 +43,8 @@ def joinpaths(*args, **kwargs):
def touch(fname):
with open(fname, "w") as _:
pass
# python closes the file when it goes out of scope
open(fname, "w").write("")
def replace(fname, find, sub):

6
src/pylorax/treebuilder.py
View File

@ -1,6 +1,6 @@
# treebuilder.py - handle arch-specific tree building stuff using templates
#
# Copyright (C) 2011-2014 Red Hat, Inc.
# Copyright (C) 2011-2015 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -324,7 +324,7 @@ def findkernels(root="/", kdir="boot"):
udev_blacklist=' !"$%&\'()*,/;<>?[\\]^`{|}~' # ASCII printable, minus whitelist
udev_blacklist += ''.join(chr(i) for i in range(32)) # ASCII non-printable
def udev_escape(label):
out = u''
out = ''
for ch in label.decode('utf8'):
out += ch if ch not in udev_blacklist else u'\\x%02x' % ord(ch)
out += ch if ch not in udev_blacklist else '\\x%02x' % ord(ch)
return out.encode('utf8')

10
src/pylorax/treeinfo.py
View File

@ -1,7 +1,7 @@
#
# treeinfo.py
#
# Copyright (C) 2010-2014 Red Hat, Inc.
# Copyright (C) 2010-2015 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -22,7 +22,7 @@
import logging
logger = logging.getLogger("pylorax.treeinfo")
import ConfigParser
import configparser
import time
@ -31,7 +31,7 @@ class TreeInfo(object):
def __init__(self, product, version, variant, basearch,
packagedir=""):
self.c = ConfigParser.ConfigParser()
self.c = configparser.ConfigParser()
section = "general"
data = {"timestamp": time.time(),
@ -43,13 +43,13 @@ class TreeInfo(object):
"packagedir": packagedir}
self.c.add_section(section)
map(lambda (key, value): self.c.set(section, key, value), data.items())
list(self.c.set(section, key, value) for key, value in data.items())
def add_section(self, section, data):
if not self.c.has_section(section):
self.c.add_section(section)
map(lambda (key, value): self.c.set(section, key, value), data.items())
list(self.c.set(section, key, value) for key, value in data.items())
def write(self, outfile):
logger.info("writing .treeinfo file")

2
tests/pylint/pylint-one.sh
View File

@ -10,7 +10,7 @@ fi
file_suffix="$(eval echo \$$#|sed s?/?_?g)"
pylint_output="$(pylint \
pylint_output="$(python3-pylint \
--msg-template='{path}:{line}: {msg_id} {symbol} {msg}' \
-r n --disable=C,R --rcfile=/dev/null \
--dummy-variables-rgx=_ \