Switch lorax to use dnf instead of yum
pylorax users will need to change to using dnf and pass a dnf.Base() object as the dbo argument instead of a yum object as the yum or ybo argument. See the lorax script for an example of how to do this. The lorax cmdline argument --excludepkgs has been removed since dnf doesn't appear to have any way to support it and packages should be controlled using templates anyway.
This commit is contained in:
parent
4b55e1eb7a
commit
431ca6cea4
@ -37,9 +37,9 @@ Requires: squashfs-tools >= 4.2
|
|||||||
Requires: util-linux
|
Requires: util-linux
|
||||||
Requires: xz
|
Requires: xz
|
||||||
Requires: pigz
|
Requires: pigz
|
||||||
Requires: yum
|
|
||||||
Requires: pykickstart
|
Requires: pykickstart
|
||||||
Requires: dracut >= 030
|
Requires: dracut >= 030
|
||||||
|
Requires: dnf
|
||||||
|
|
||||||
%if 0%{?fedora}
|
%if 0%{?fedora}
|
||||||
# Fedora specific deps
|
# Fedora specific deps
|
||||||
|
@ -36,7 +36,7 @@ import selinux
|
|||||||
from pylorax.base import BaseLoraxClass, DataHolder
|
from pylorax.base import BaseLoraxClass, DataHolder
|
||||||
import pylorax.output as output
|
import pylorax.output as output
|
||||||
|
|
||||||
import yum
|
import dnf
|
||||||
|
|
||||||
from pylorax.sysutils import joinpaths, remove, linktree
|
from pylorax.sysutils import joinpaths, remove, linktree
|
||||||
from rpmUtils.arch import getBaseArch
|
from rpmUtils.arch import getBaseArch
|
||||||
@ -96,9 +96,6 @@ class Lorax(BaseLoraxClass):
|
|||||||
self.conf.add_section("templates")
|
self.conf.add_section("templates")
|
||||||
self.conf.set("templates", "ramdisk", "ramdisk.ltmpl")
|
self.conf.set("templates", "ramdisk", "ramdisk.ltmpl")
|
||||||
|
|
||||||
self.conf.add_section("yum")
|
|
||||||
self.conf.set("yum", "skipbroken", "0")
|
|
||||||
|
|
||||||
self.conf.add_section("compression")
|
self.conf.add_section("compression")
|
||||||
self.conf.set("compression", "type", "xz")
|
self.conf.set("compression", "type", "xz")
|
||||||
self.conf.set("compression", "args", "")
|
self.conf.set("compression", "args", "")
|
||||||
@ -149,7 +146,7 @@ class Lorax(BaseLoraxClass):
|
|||||||
fh.setLevel(logging.DEBUG)
|
fh.setLevel(logging.DEBUG)
|
||||||
logger.addHandler(fh)
|
logger.addHandler(fh)
|
||||||
|
|
||||||
def run(self, ybo, product, version, release, variant="", bugurl="",
|
def run(self, dbo, product, version, release, variant="", bugurl="",
|
||||||
isfinal=False, workdir=None, outputdir=None, buildarch=None, volid=None,
|
isfinal=False, workdir=None, outputdir=None, buildarch=None, volid=None,
|
||||||
domacboot=True, doupgrade=True, remove_temp=False,
|
domacboot=True, doupgrade=True, remove_temp=False,
|
||||||
installpkgs=None,
|
installpkgs=None,
|
||||||
@ -221,16 +218,16 @@ class Lorax(BaseLoraxClass):
|
|||||||
logger.critical("selinux must be disabled or in Permissive mode")
|
logger.critical("selinux must be disabled or in Permissive mode")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
# do we have a proper yum base object?
|
# do we have a proper dnf base object?
|
||||||
logger.info("checking yum base object")
|
logger.info("checking dnf base object")
|
||||||
if not isinstance(ybo, yum.YumBase):
|
if not isinstance(dbo, dnf.Base):
|
||||||
logger.critical("no yum base object")
|
logger.critical("no dnf base object")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
self.inroot = ybo.conf.installroot
|
self.inroot = dbo.conf.installroot
|
||||||
logger.debug("using install root: {0}".format(self.inroot))
|
logger.debug("using install root: {0}".format(self.inroot))
|
||||||
|
|
||||||
if not buildarch:
|
if not buildarch:
|
||||||
buildarch = get_buildarch(ybo)
|
buildarch = get_buildarch(dbo)
|
||||||
|
|
||||||
logger.info("setting up build architecture")
|
logger.info("setting up build architecture")
|
||||||
self.arch = ArchData(buildarch)
|
self.arch = ArchData(buildarch)
|
||||||
@ -253,15 +250,14 @@ class Lorax(BaseLoraxClass):
|
|||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
templatedir = self.conf.get("lorax", "sharedir")
|
templatedir = self.conf.get("lorax", "sharedir")
|
||||||
# NOTE: rb.root = ybo.conf.installroot (== self.inroot)
|
# NOTE: rb.root = dbo.conf.installroot (== self.inroot)
|
||||||
rb = RuntimeBuilder(product=self.product, arch=self.arch,
|
rb = RuntimeBuilder(product=self.product, arch=self.arch,
|
||||||
yum=ybo, templatedir=templatedir,
|
dbo=dbo, templatedir=templatedir,
|
||||||
installpkgs=installpkgs,
|
installpkgs=installpkgs,
|
||||||
add_templates=add_templates,
|
add_templates=add_templates,
|
||||||
add_template_vars=add_template_vars)
|
add_template_vars=add_template_vars)
|
||||||
|
|
||||||
logger.info("installing runtime packages")
|
logger.info("installing runtime packages")
|
||||||
rb.yum.conf.skip_broken = self.conf.getboolean("yum", "skipbroken")
|
|
||||||
rb.install()
|
rb.install()
|
||||||
|
|
||||||
# write .buildstamp
|
# write .buildstamp
|
||||||
@ -358,10 +354,12 @@ class Lorax(BaseLoraxClass):
|
|||||||
remove(self.workdir)
|
remove(self.workdir)
|
||||||
|
|
||||||
|
|
||||||
def get_buildarch(ybo):
|
def get_buildarch(dbo):
|
||||||
# get architecture of the available anaconda package
|
# get architecture of the available anaconda package
|
||||||
buildarch = None
|
buildarch = None
|
||||||
for anaconda in ybo.doPackageLists(patterns=["anaconda"]).available:
|
q = dbo.sack.query()
|
||||||
|
a = q.available()
|
||||||
|
for anaconda in a.filter(name="anaconda"):
|
||||||
if anaconda.arch != "src":
|
if anaconda.arch != "src":
|
||||||
buildarch = anaconda.arch
|
buildarch = anaconda.arch
|
||||||
break
|
break
|
||||||
|
107
src/pylorax/dnfhelper.py
Normal file
107
src/pylorax/dnfhelper.py
Normal file
@ -0,0 +1,107 @@
|
|||||||
|
#
|
||||||
|
# dnfhelper.py
|
||||||
|
#
|
||||||
|
# Copyright (C) 2010-2014 Red Hat, Inc.
|
||||||
|
#
|
||||||
|
# This program is free software; you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU General Public License as published by
|
||||||
|
# the Free Software Foundation; either version 2 of the License, or
|
||||||
|
# (at your option) any later version.
|
||||||
|
#
|
||||||
|
# This program is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU General Public License
|
||||||
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
#
|
||||||
|
# Red Hat Author(s): Martin Gracik <mgracik@redhat.com>
|
||||||
|
# Brian C. Lane <bcl@redhat.com>
|
||||||
|
#
|
||||||
|
|
||||||
|
import logging
|
||||||
|
logger = logging.getLogger("pylorax.dnfhelper")
|
||||||
|
import dnf
|
||||||
|
import collections
|
||||||
|
import time
|
||||||
|
import pylorax.output as output
|
||||||
|
|
||||||
|
__all__ = ['LoraxDownloadCallback', 'LoraxRpmCallback']
|
||||||
|
|
||||||
|
def _paced(fn):
|
||||||
|
"""Execute `fn` no more often then every 2 seconds."""
|
||||||
|
def paced_fn(self, *args):
|
||||||
|
now = time.time()
|
||||||
|
if now - self.last_time < 2:
|
||||||
|
return
|
||||||
|
self.last_time = now
|
||||||
|
return fn(self, *args)
|
||||||
|
return paced_fn
|
||||||
|
|
||||||
|
|
||||||
|
class LoraxDownloadCallback(dnf.callback.DownloadProgress):
|
||||||
|
def __init__(self):
|
||||||
|
self.downloads = collections.defaultdict(int)
|
||||||
|
self.last_time = time.time()
|
||||||
|
self.total_files = 0
|
||||||
|
self.total_size = 0
|
||||||
|
|
||||||
|
self.pkgno = 0
|
||||||
|
self.total = 0
|
||||||
|
|
||||||
|
self.output = output.LoraxOutput()
|
||||||
|
|
||||||
|
@_paced
|
||||||
|
def _update(self):
|
||||||
|
msg = "Downloading %(pkgno)s / %(total_files)s RPMs, " \
|
||||||
|
"%(downloaded)s / %(total_size)s (%(percent)d%%) done.\n"
|
||||||
|
downloaded = sum(self.downloads.values())
|
||||||
|
vals = {
|
||||||
|
'downloaded' : downloaded,
|
||||||
|
'percent' : int(100 * downloaded/self.total_size),
|
||||||
|
'pkgno' : self.pkgno,
|
||||||
|
'total_files' : self.total_files,
|
||||||
|
'total_size' : self.total_size
|
||||||
|
}
|
||||||
|
self.output.write(msg % vals)
|
||||||
|
|
||||||
|
def end(self, payload, status, err_msg):
|
||||||
|
nevra = str(payload)
|
||||||
|
if status is dnf.callback.STATUS_OK:
|
||||||
|
self.downloads[nevra] = payload.download_size
|
||||||
|
self.pkgno += 1
|
||||||
|
self._update()
|
||||||
|
return
|
||||||
|
logger.critical("Failed to download '%s': %d - %s", nevra, status, err_msg)
|
||||||
|
|
||||||
|
def progress(self, payload, done):
|
||||||
|
nevra = str(payload)
|
||||||
|
self.downloads[nevra] = done
|
||||||
|
self._update()
|
||||||
|
|
||||||
|
def start(self, total_files, total_size):
|
||||||
|
self.total_files = total_files
|
||||||
|
self.total_size = total_size
|
||||||
|
|
||||||
|
|
||||||
|
class LoraxRpmCallback(dnf.callback.LoggingTransactionDisplay):
|
||||||
|
def __init__(self, queue):
|
||||||
|
super(LoraxRpmCallback, self).__init__()
|
||||||
|
self._queue = queue
|
||||||
|
self._last_ts = None
|
||||||
|
self.cnt = 0
|
||||||
|
|
||||||
|
def event(self, package, action, te_current, te_total, ts_current, ts_total):
|
||||||
|
if action == self.PKG_INSTALL and te_current == 0:
|
||||||
|
# do not report same package twice
|
||||||
|
if self._last_ts == ts_current:
|
||||||
|
return
|
||||||
|
self._last_ts = ts_current
|
||||||
|
|
||||||
|
msg = '(%d/%d) %s.%s' % \
|
||||||
|
(ts_current, ts_total, package.name, package.arch)
|
||||||
|
self.cnt += 1
|
||||||
|
self._queue.put(('install', msg))
|
||||||
|
elif action == self.TRANS_POST:
|
||||||
|
self._queue.put(('post', None))
|
@ -28,15 +28,18 @@ from os.path import basename, isdir
|
|||||||
from subprocess import CalledProcessError
|
from subprocess import CalledProcessError
|
||||||
|
|
||||||
from pylorax.sysutils import joinpaths, cpfile, mvfile, replace, remove
|
from pylorax.sysutils import joinpaths, cpfile, mvfile, replace, remove
|
||||||
from pylorax.yumhelper import LoraxDownloadCallback, LoraxTransactionCallback, LoraxRpmCallback
|
from pylorax.dnfhelper import LoraxDownloadCallback, LoraxRpmCallback
|
||||||
from pylorax.base import DataHolder
|
from pylorax.base import DataHolder
|
||||||
from pylorax.executils import runcmd, runcmd_output
|
from pylorax.executils import runcmd, runcmd_output
|
||||||
from pylorax.imgutils import mkcpio
|
from pylorax.imgutils import mkcpio
|
||||||
|
import pylorax.output as output
|
||||||
|
|
||||||
from mako.lookup import TemplateLookup
|
from mako.lookup import TemplateLookup
|
||||||
from mako.exceptions import text_error_template
|
from mako.exceptions import text_error_template
|
||||||
import sys, traceback
|
import sys, traceback
|
||||||
import struct
|
import struct
|
||||||
|
import dnf
|
||||||
|
import multiprocessing
|
||||||
|
|
||||||
class LoraxTemplate(object):
|
class LoraxTemplate(object):
|
||||||
def __init__(self, directories=None):
|
def __init__(self, directories=None):
|
||||||
@ -108,7 +111,7 @@ class LoraxTemplateRunner(object):
|
|||||||
This class parses and executes Lorax templates. Sample usage:
|
This class parses and executes Lorax templates. Sample usage:
|
||||||
|
|
||||||
# install a bunch of packages
|
# install a bunch of packages
|
||||||
runner = LoraxTemplateRunner(inroot=rundir, outroot=rundir, yum=yum_obj)
|
runner = LoraxTemplateRunner(inroot=rundir, outroot=rundir, dbo=dnf_obj)
|
||||||
runner.run("install-packages.ltmpl")
|
runner.run("install-packages.ltmpl")
|
||||||
|
|
||||||
# modify a runtime dir
|
# modify a runtime dir
|
||||||
@ -145,11 +148,11 @@ class LoraxTemplateRunner(object):
|
|||||||
|
|
||||||
* Commands should raise exceptions for errors - don't use sys.exit()
|
* Commands should raise exceptions for errors - don't use sys.exit()
|
||||||
'''
|
'''
|
||||||
def __init__(self, inroot, outroot, yum=None, fatalerrors=True,
|
def __init__(self, inroot, outroot, dbo=None, fatalerrors=True,
|
||||||
templatedir=None, defaults=None):
|
templatedir=None, defaults=None):
|
||||||
self.inroot = inroot
|
self.inroot = inroot
|
||||||
self.outroot = outroot
|
self.outroot = outroot
|
||||||
self.yum = yum
|
self.dbo = dbo
|
||||||
self.fatalerrors = fatalerrors
|
self.fatalerrors = fatalerrors
|
||||||
self.templatedir = templatedir or "/usr/share/lorax"
|
self.templatedir = templatedir or "/usr/share/lorax"
|
||||||
self.templatefile = None
|
self.templatefile = None
|
||||||
@ -166,8 +169,8 @@ class LoraxTemplateRunner(object):
|
|||||||
return joinpaths(self.inroot, path)
|
return joinpaths(self.inroot, path)
|
||||||
|
|
||||||
def _filelist(self, *pkgs):
|
def _filelist(self, *pkgs):
|
||||||
pkglist = self.yum.doPackageLists(pkgnarrow="installed", patterns=pkgs)
|
pkglist = self.dbo.doPackageLists(pkgnarrow="installed", patterns=pkgs)
|
||||||
return set([f for pkg in pkglist.installed for f in pkg.filelist+pkg.ghostlist])
|
return set([f for pkg in pkglist for f in pkg.files])
|
||||||
|
|
||||||
def _getsize(self, *files):
|
def _getsize(self, *files):
|
||||||
return sum(os.path.getsize(self._out(f)) for f in files if os.path.isfile(self._out(f)))
|
return sum(os.path.getsize(self._out(f)) for f in files if os.path.isfile(self._out(f)))
|
||||||
@ -447,9 +450,9 @@ class LoraxTemplateRunner(object):
|
|||||||
cmd = cmd[1:]
|
cmd = cmd[1:]
|
||||||
|
|
||||||
try:
|
try:
|
||||||
output = runcmd_output(cmd, cwd=cwd)
|
stdout = runcmd_output(cmd, cwd=cwd)
|
||||||
if output:
|
if stdout:
|
||||||
logger.debug('command output:\n%s', output)
|
logger.debug('command output:\n%s', stdout)
|
||||||
logger.debug("command finished successfully")
|
logger.debug("command finished successfully")
|
||||||
except CalledProcessError as e:
|
except CalledProcessError as e:
|
||||||
if e.output:
|
if e.output:
|
||||||
@ -471,10 +474,10 @@ class LoraxTemplateRunner(object):
|
|||||||
|
|
||||||
for p in pkgs:
|
for p in pkgs:
|
||||||
try:
|
try:
|
||||||
self.yum.install(pattern=p)
|
self.dbo.install(p)
|
||||||
except Exception as e: # pylint: disable=broad-except
|
except Exception as e: # pylint: disable=broad-except
|
||||||
# FIXME: save exception and re-raise after the loop finishes
|
# FIXME: save exception and re-raise after the loop finishes
|
||||||
logger.error("installpkg %s failed: %s",p,str(e))
|
logger.error("installpkg %s failed: %s", p, str(e))
|
||||||
if required:
|
if required:
|
||||||
raise
|
raise
|
||||||
|
|
||||||
@ -501,19 +504,60 @@ class LoraxTemplateRunner(object):
|
|||||||
Actually install all the packages requested by previous 'installpkg'
|
Actually install all the packages requested by previous 'installpkg'
|
||||||
commands.
|
commands.
|
||||||
'''
|
'''
|
||||||
self.yum.buildTransaction()
|
|
||||||
dl_callback = LoraxDownloadCallback()
|
def do_transaction(base, queue):
|
||||||
self.yum.repos.setProgressBar(dl_callback)
|
try:
|
||||||
self.yum.processTransaction(callback=LoraxTransactionCallback(dl_callback),
|
display = LoraxRpmCallback(queue)
|
||||||
rpmDisplay=LoraxRpmCallback())
|
base.do_transaction(display=display)
|
||||||
|
except BaseException as e:
|
||||||
|
logger.error("The transaction process has ended abruptly: %s", e)
|
||||||
|
queue.put(('quit', str(e)))
|
||||||
|
|
||||||
|
self.dbo.reset()
|
||||||
|
try:
|
||||||
|
logger.info("Checking dependencies")
|
||||||
|
self.dbo.resolve()
|
||||||
|
except dnf.exceptions.DepsolveError as e:
|
||||||
|
logger.error("Dependency check failed: %s", e)
|
||||||
|
raise
|
||||||
|
logger.info("%d packages selected", len(self.dbo.transaction))
|
||||||
|
if len(self.dbo.transaction) == 0:
|
||||||
|
raise Exception("No packages in transaction")
|
||||||
|
|
||||||
|
pkgs_to_download = self.dbo.transaction.install_set
|
||||||
|
logger.info("Downloading packages")
|
||||||
|
progress = LoraxDownloadCallback()
|
||||||
|
try:
|
||||||
|
self.dbo.download_packages(pkgs_to_download, progress)
|
||||||
|
except dnf.exceptions.DownloadError as e:
|
||||||
|
logger.error("Failed to download the following packages: %s", e)
|
||||||
|
raise
|
||||||
|
|
||||||
|
logger.info("Preparing transaction from installation source")
|
||||||
|
queue = multiprocessing.Queue()
|
||||||
|
msgout = output.LoraxOutput()
|
||||||
|
process = multiprocessing.Process(target=do_transaction, args=(self.dbo, queue))
|
||||||
|
process.start()
|
||||||
|
(token, msg) = queue.get()
|
||||||
|
while token not in ('post', 'quit'):
|
||||||
|
if token == 'install':
|
||||||
|
logging.info("%s", msg)
|
||||||
|
msgout.writeline(msg)
|
||||||
|
(token, msg) = queue.get()
|
||||||
|
if token == 'quit':
|
||||||
|
logger.error("Transaction failed.")
|
||||||
|
raise Exception("Transaction failed")
|
||||||
|
|
||||||
|
logger.info("Performing post-installation setup tasks")
|
||||||
|
process.join()
|
||||||
|
|
||||||
# verify if all packages that were supposed to be installed,
|
# verify if all packages that were supposed to be installed,
|
||||||
# are really installed
|
# are really installed
|
||||||
errs = [t.po for t in self.yum.tsInfo if not self.yum.rpmdb.contains(po=t.po)]
|
errs = [t.po for t in self.dbo.tsInfo if not self.dbo.rpmdb.contains(po=t.po)]
|
||||||
for po in errs:
|
for po in errs:
|
||||||
logger.error("package '%s' was not installed", po)
|
logger.error("package '%s' was not installed", po)
|
||||||
|
|
||||||
self.yum.closeRpmDB()
|
self.dbo.close()
|
||||||
|
|
||||||
def removefrom(self, pkg, *globs):
|
def removefrom(self, pkg, *globs):
|
||||||
'''
|
'''
|
||||||
|
@ -67,19 +67,19 @@ def generate_module_info(moddir, outfile=None):
|
|||||||
|
|
||||||
class RuntimeBuilder(object):
|
class RuntimeBuilder(object):
|
||||||
'''Builds the anaconda runtime image.'''
|
'''Builds the anaconda runtime image.'''
|
||||||
def __init__(self, product, arch, yum, templatedir=None,
|
def __init__(self, product, arch, dbo, templatedir=None,
|
||||||
installpkgs=None,
|
installpkgs=None,
|
||||||
add_templates=None,
|
add_templates=None,
|
||||||
add_template_vars=None):
|
add_template_vars=None):
|
||||||
root = yum.conf.installroot
|
root = dbo.conf.installroot
|
||||||
# use a copy of product so we can modify it locally
|
# use a copy of product so we can modify it locally
|
||||||
product = product.copy()
|
product = product.copy()
|
||||||
product.name = product.name.lower()
|
product.name = product.name.lower()
|
||||||
self.vars = DataHolder(arch=arch, product=product, yum=yum, root=root,
|
self.vars = DataHolder(arch=arch, product=product, dbo=dbo, root=root,
|
||||||
basearch=arch.basearch, libdir=arch.libdir)
|
basearch=arch.basearch, libdir=arch.libdir)
|
||||||
self.yum = yum
|
self.dbo = dbo
|
||||||
self._runner = LoraxTemplateRunner(inroot=root, outroot=root,
|
self._runner = LoraxTemplateRunner(inroot=root, outroot=root,
|
||||||
yum=yum, templatedir=templatedir)
|
dbo=dbo, templatedir=templatedir)
|
||||||
self.add_templates = add_templates or []
|
self.add_templates = add_templates or []
|
||||||
self.add_template_vars = add_template_vars or {}
|
self.add_template_vars = add_template_vars or {}
|
||||||
self._installpkgs = installpkgs or []
|
self._installpkgs = installpkgs or []
|
||||||
@ -87,7 +87,9 @@ class RuntimeBuilder(object):
|
|||||||
|
|
||||||
def _install_branding(self):
|
def _install_branding(self):
|
||||||
release = None
|
release = None
|
||||||
for pkg in self.yum.whatProvides('/etc/system-release', None, None):
|
q = self.dbo.sack.query()
|
||||||
|
a = q.available()
|
||||||
|
for pkg in a.filter(provides='/etc/system-release'):
|
||||||
if pkg.name.startswith('generic'):
|
if pkg.name.startswith('generic'):
|
||||||
continue
|
continue
|
||||||
else:
|
else:
|
||||||
@ -119,9 +121,10 @@ class RuntimeBuilder(object):
|
|||||||
'''debugging data: write out lists of package contents'''
|
'''debugging data: write out lists of package contents'''
|
||||||
if not os.path.isdir(pkglistdir):
|
if not os.path.isdir(pkglistdir):
|
||||||
os.makedirs(pkglistdir)
|
os.makedirs(pkglistdir)
|
||||||
for pkgobj in self.yum.doPackageLists(pkgnarrow='installed').installed:
|
q = self.dbo.sack.query()
|
||||||
|
for pkgobj in q.installed():
|
||||||
with open(joinpaths(pkglistdir, pkgobj.name), "w") as fobj:
|
with open(joinpaths(pkglistdir, pkgobj.name), "w") as fobj:
|
||||||
for fname in pkgobj.filelist + pkgobj.dirlist:
|
for fname in pkgobj.files:
|
||||||
fobj.write("{0}\n".format(fname))
|
fobj.write("{0}\n".format(fname))
|
||||||
|
|
||||||
def postinstall(self):
|
def postinstall(self):
|
||||||
@ -143,8 +146,9 @@ class RuntimeBuilder(object):
|
|||||||
'''debugging data: write a big list of pkg sizes'''
|
'''debugging data: write a big list of pkg sizes'''
|
||||||
fobj = open(pkgsizefile, "w")
|
fobj = open(pkgsizefile, "w")
|
||||||
getsize = lambda f: os.lstat(f).st_size if os.path.exists(f) else 0
|
getsize = lambda f: os.lstat(f).st_size if os.path.exists(f) else 0
|
||||||
for p in sorted(self.yum.doPackageLists(pkgnarrow='installed').installed):
|
q = self.dbo.sack.query()
|
||||||
pkgsize = sum(getsize(joinpaths(self.vars.root,f)) for f in p.filelist)
|
for p in sorted(q.installed()):
|
||||||
|
pkgsize = sum(getsize(joinpaths(self.vars.root,f)) for f in p.files)
|
||||||
fobj.write("{0.name}.{0.arch}: {1}\n".format(p, pkgsize))
|
fobj.write("{0.name}.{0.arch}: {1}\n".format(p, pkgsize))
|
||||||
|
|
||||||
def generate_module_data(self):
|
def generate_module_data(self):
|
||||||
|
@ -1,127 +0,0 @@
|
|||||||
#
|
|
||||||
# yumhelper.py
|
|
||||||
#
|
|
||||||
# Copyright (C) 2010-2014 Red Hat, Inc.
|
|
||||||
#
|
|
||||||
# This program is free software; you can redistribute it and/or modify
|
|
||||||
# it under the terms of the GNU General Public License as published by
|
|
||||||
# the Free Software Foundation; either version 2 of the License, or
|
|
||||||
# (at your option) any later version.
|
|
||||||
#
|
|
||||||
# This program is distributed in the hope that it will be useful,
|
|
||||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
# GNU General Public License for more details.
|
|
||||||
#
|
|
||||||
# You should have received a copy of the GNU General Public License
|
|
||||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
#
|
|
||||||
# Red Hat Author(s): Martin Gracik <mgracik@redhat.com>
|
|
||||||
#
|
|
||||||
|
|
||||||
import logging
|
|
||||||
logger = logging.getLogger("pylorax.yumhelper")
|
|
||||||
import sys
|
|
||||||
import yum, yum.callbacks, yum.rpmtrans
|
|
||||||
import pylorax.output as output
|
|
||||||
|
|
||||||
__all__ = ['LoraxDownloadCallback', 'LoraxTransactionCallback',
|
|
||||||
'LoraxRpmCallback']
|
|
||||||
|
|
||||||
class LoraxDownloadCallback(yum.callbacks.DownloadBaseCallback):
|
|
||||||
def __init__(self):
|
|
||||||
yum.callbacks.DownloadBaseCallback.__init__(self)
|
|
||||||
|
|
||||||
self.pkgno = 0
|
|
||||||
self.total = 0
|
|
||||||
|
|
||||||
self.output = output.LoraxOutput()
|
|
||||||
|
|
||||||
|
|
||||||
def updateProgress(self, name, frac, fread, ftime):
|
|
||||||
"""
|
|
||||||
Update the progress bar
|
|
||||||
@param name: filename
|
|
||||||
@param frac: progress fraction (0 -> 1)
|
|
||||||
@param fread: formated string containing BytesRead
|
|
||||||
@param ftime: formated string containing remaining or elapsed time
|
|
||||||
"""
|
|
||||||
# Only update when it is finished downloading
|
|
||||||
if frac < 1:
|
|
||||||
return
|
|
||||||
|
|
||||||
self.pkgno += 1
|
|
||||||
info = "({0:3d}/{1:3d}) "
|
|
||||||
info = info.format(self.pkgno, self.total)
|
|
||||||
|
|
||||||
infolen, pkglen = len(info), len(name)
|
|
||||||
if (infolen + pkglen) > self.output.width:
|
|
||||||
name = "{0}...".format(name[:self.output.width-infolen-3])
|
|
||||||
|
|
||||||
msg = "{0}<b>{1}</b>\n".format(info, name)
|
|
||||||
self.output.write(msg)
|
|
||||||
|
|
||||||
|
|
||||||
class LoraxTransactionCallback(object):
|
|
||||||
|
|
||||||
def __init__(self, dl_callback):
|
|
||||||
self.output = output.LoraxOutput()
|
|
||||||
|
|
||||||
self.dl_callback = dl_callback
|
|
||||||
|
|
||||||
def event(self, state, data=None):
|
|
||||||
if state == yum.callbacks.PT_DOWNLOAD:
|
|
||||||
self.output.write("downloading packages\n")
|
|
||||||
elif state == yum.callbacks.PT_DOWNLOAD_PKGS:
|
|
||||||
# Initialize the total number of packages being downloaded
|
|
||||||
self.dl_callback.total = len(data)
|
|
||||||
elif state == yum.callbacks.PT_GPGCHECK:
|
|
||||||
self.output.write("checking package signatures\n")
|
|
||||||
elif state == yum.callbacks.PT_TEST_TRANS:
|
|
||||||
self.output.write("running test transaction\n")
|
|
||||||
elif state == yum.callbacks.PT_TRANSACTION:
|
|
||||||
self.output.write("running transaction\n")
|
|
||||||
|
|
||||||
|
|
||||||
class LoraxRpmCallback(yum.rpmtrans.RPMBaseCallback):
|
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
yum.rpmtrans.RPMBaseCallback.__init__(self)
|
|
||||||
self.output = output.LoraxOutput()
|
|
||||||
|
|
||||||
def event(self, package, action, te_current, te_total,
|
|
||||||
ts_current, ts_total):
|
|
||||||
|
|
||||||
action_str = self.action[action].encode("utf-8")
|
|
||||||
info = "({0:3d}/{1:3d}) [{2:3.0f}%] {3} "
|
|
||||||
info = info.format(ts_current, ts_total,
|
|
||||||
float(te_current) / float(te_total) * 100,
|
|
||||||
action_str.lower())
|
|
||||||
|
|
||||||
pkg = "{0}".format(package)
|
|
||||||
|
|
||||||
infolen, pkglen = len(info), len(pkg)
|
|
||||||
if (infolen + pkglen) > self.output.width:
|
|
||||||
pkg = "{0}...".format(pkg[:self.output.width-infolen-3])
|
|
||||||
|
|
||||||
msg = "{0}<b>{1}</b>".format(info, pkg)
|
|
||||||
|
|
||||||
# When not outputting to a tty we only want to print it once at the end
|
|
||||||
if sys.stdout.isatty():
|
|
||||||
self.output.write(msg + "\r")
|
|
||||||
if te_current == te_total:
|
|
||||||
self.output.write("\n")
|
|
||||||
elif te_current == te_total:
|
|
||||||
self.output.write(msg + "\n")
|
|
||||||
|
|
||||||
|
|
||||||
def filelog(self, package, action):
|
|
||||||
if self.fileaction.get(action) == "Installed":
|
|
||||||
logger.debug("{0} installed successfully".format(package))
|
|
||||||
|
|
||||||
def errorlog(self, msg):
|
|
||||||
logger.warning("RPM transaction error: %s", msg)
|
|
||||||
|
|
||||||
def scriptout(self, package, msgs):
|
|
||||||
if msgs:
|
|
||||||
logger.info("%s scriptlet output:\n%s", package, msgs)
|
|
161
src/sbin/lorax
161
src/sbin/lorax
@ -25,20 +25,16 @@ import logging
|
|||||||
log = logging.getLogger("lorax")
|
log = logging.getLogger("lorax")
|
||||||
program_log = logging.getLogger("program")
|
program_log = logging.getLogger("program")
|
||||||
pylorax_log = logging.getLogger("pylorax")
|
pylorax_log = logging.getLogger("pylorax")
|
||||||
yum_log = logging.getLogger("yum")
|
dnf_log = logging.getLogger("dnf")
|
||||||
|
|
||||||
|
|
||||||
import sys
|
import sys
|
||||||
import os
|
import os
|
||||||
import tempfile
|
import tempfile
|
||||||
from optparse import OptionParser, OptionGroup
|
from optparse import OptionParser, OptionGroup
|
||||||
import ConfigParser
|
|
||||||
import shutil
|
import shutil
|
||||||
|
|
||||||
import yum
|
import dnf
|
||||||
# This is a bit of a hack to short circuit yum's internal logging
|
|
||||||
# handler setup. We already set one up so we don't need it to run.
|
|
||||||
yum.logginglevels._added_handlers = True
|
|
||||||
import pylorax
|
import pylorax
|
||||||
|
|
||||||
def setup_logging(opts):
|
def setup_logging(opts):
|
||||||
@ -67,12 +63,12 @@ def setup_logging(opts):
|
|||||||
fh.setLevel(logging.DEBUG)
|
fh.setLevel(logging.DEBUG)
|
||||||
program_log.addHandler(fh)
|
program_log.addHandler(fh)
|
||||||
|
|
||||||
# yum logging
|
# dnf logging
|
||||||
yum_log.setLevel(logging.DEBUG)
|
dnf_log.setLevel(logging.DEBUG)
|
||||||
logfile = os.path.abspath(os.path.dirname(opts.logfile))+"/yum.log"
|
logfile = os.path.abspath(os.path.dirname(opts.logfile))+"/dnf.log"
|
||||||
fh = logging.FileHandler(filename=logfile, mode="w")
|
fh = logging.FileHandler(filename=logfile, mode="w")
|
||||||
fh.setLevel(logging.DEBUG)
|
fh.setLevel(logging.DEBUG)
|
||||||
yum_log.addHandler(fh)
|
dnf_log.addHandler(fh)
|
||||||
|
|
||||||
|
|
||||||
def main(args):
|
def main(args):
|
||||||
@ -116,9 +112,6 @@ def main(args):
|
|||||||
help="config file", metavar="STRING")
|
help="config file", metavar="STRING")
|
||||||
optional.add_option("--proxy", default=None,
|
optional.add_option("--proxy", default=None,
|
||||||
help="repo proxy url:port", metavar="STRING")
|
help="repo proxy url:port", metavar="STRING")
|
||||||
optional.add_option("-e", "--excludepkgs", default=[],
|
|
||||||
action="append", metavar="STRING",
|
|
||||||
help="package glob to exclude (may be listed multiple times)")
|
|
||||||
optional.add_option("-i", "--installpkgs", default=[],
|
optional.add_option("-i", "--installpkgs", default=[],
|
||||||
action="append", metavar="STRING",
|
action="append", metavar="STRING",
|
||||||
help="package glob to install before runtime-install.tmpl runs. (may be listed multiple times)")
|
help="package glob to install before runtime-install.tmpl runs. (may be listed multiple times)")
|
||||||
@ -180,17 +173,16 @@ def main(args):
|
|||||||
# create the temporary directory for lorax
|
# create the temporary directory for lorax
|
||||||
tempdir = tempfile.mkdtemp(prefix="lorax.", dir=tempfile.gettempdir())
|
tempdir = tempfile.mkdtemp(prefix="lorax.", dir=tempfile.gettempdir())
|
||||||
|
|
||||||
# create the yumbase object
|
|
||||||
installtree = os.path.join(tempdir, "installtree")
|
installtree = os.path.join(tempdir, "installtree")
|
||||||
os.mkdir(installtree)
|
os.mkdir(installtree)
|
||||||
yumtempdir = os.path.join(tempdir, "yum")
|
dnftempdir = os.path.join(tempdir, "dnf")
|
||||||
os.mkdir(yumtempdir)
|
os.mkdir(dnftempdir)
|
||||||
|
|
||||||
yb = get_yum_base_object(installtree, opts.source, opts.mirrorlist,
|
dnfbase = get_dnf_base_object(installtree, opts.source, opts.mirrorlist,
|
||||||
yumtempdir, opts.proxy, opts.excludepkgs)
|
dnftempdir, opts.proxy, opts.version)
|
||||||
|
|
||||||
if yb is None:
|
if dnfbase is None:
|
||||||
print("error: unable to create the yumbase object", file=sys.stderr)
|
print("error: unable to create the dnf base object", file=sys.stderr)
|
||||||
shutil.rmtree(tempdir)
|
shutil.rmtree(tempdir)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
@ -204,7 +196,7 @@ def main(args):
|
|||||||
# run lorax
|
# run lorax
|
||||||
lorax = pylorax.Lorax()
|
lorax = pylorax.Lorax()
|
||||||
lorax.configure(conf_file=opts.config)
|
lorax.configure(conf_file=opts.config)
|
||||||
lorax.run(yb, opts.product, opts.version, opts.release,
|
lorax.run(dnfbase, opts.product, opts.version, opts.release,
|
||||||
opts.variant, opts.bugurl, opts.isfinal,
|
opts.variant, opts.bugurl, opts.isfinal,
|
||||||
workdir=tempdir, outputdir=outputdir, buildarch=opts.buildarch,
|
workdir=tempdir, outputdir=outputdir, buildarch=opts.buildarch,
|
||||||
volid=opts.volid, domacboot=opts.domacboot, doupgrade=opts.doupgrade,
|
volid=opts.volid, domacboot=opts.domacboot, doupgrade=opts.doupgrade,
|
||||||
@ -214,9 +206,17 @@ def main(args):
|
|||||||
remove_temp=True)
|
remove_temp=True)
|
||||||
|
|
||||||
|
|
||||||
def get_yum_base_object(installroot, repositories, mirrorlists=None,
|
def get_dnf_base_object(installroot, repositories, mirrorlists=None,
|
||||||
tempdir="/var/tmp", proxy=None, excludepkgs=None):
|
tempdir="/var/tmp", proxy=None, releasever="21"):
|
||||||
|
""" Create a dnf Base object and setup the repositories and installroot
|
||||||
|
|
||||||
|
:param string installroot: Full path to the installroot
|
||||||
|
:param list repositories: List of repositories to use for the installation
|
||||||
|
:param list mirrorlist: List of mirrors to use
|
||||||
|
:param string tempdir: Path of temporary directory
|
||||||
|
:param string proxy: http proxy to use when fetching packages
|
||||||
|
:param string releasever: Release version to pass to dnf
|
||||||
|
"""
|
||||||
def sanitize_repo(repo):
|
def sanitize_repo(repo):
|
||||||
"""Convert bare paths to file:/// URIs, and silently reject protocols unhandled by yum"""
|
"""Convert bare paths to file:/// URIs, and silently reject protocols unhandled by yum"""
|
||||||
if repo.startswith("/"):
|
if repo.startswith("/"):
|
||||||
@ -227,7 +227,6 @@ def get_yum_base_object(installroot, repositories, mirrorlists=None,
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
mirrorlists = mirrorlists or []
|
mirrorlists = mirrorlists or []
|
||||||
excludepkgs = excludepkgs or []
|
|
||||||
|
|
||||||
# sanitize the repositories
|
# sanitize the repositories
|
||||||
repositories = map(sanitize_repo, repositories)
|
repositories = map(sanitize_repo, repositories)
|
||||||
@ -237,78 +236,70 @@ def get_yum_base_object(installroot, repositories, mirrorlists=None,
|
|||||||
repositories = filter(bool, repositories)
|
repositories = filter(bool, repositories)
|
||||||
mirrorlists = filter(bool, mirrorlists)
|
mirrorlists = filter(bool, mirrorlists)
|
||||||
|
|
||||||
cachedir = os.path.join(tempdir, "yum.cache")
|
cachedir = os.path.join(tempdir, "dnf.cache")
|
||||||
if not os.path.isdir(cachedir):
|
if not os.path.isdir(cachedir):
|
||||||
os.mkdir(cachedir)
|
os.mkdir(cachedir)
|
||||||
|
|
||||||
yumconf = os.path.join(tempdir, "yum.conf")
|
logdir = os.path.join(tempdir, "dnf.logs")
|
||||||
c = ConfigParser.ConfigParser()
|
if not os.path.isdir(logdir):
|
||||||
|
os.mkdir(logdir)
|
||||||
|
|
||||||
# add the main section
|
dnfbase = dnf.Base()
|
||||||
section = "main"
|
conf = dnfbase.conf
|
||||||
data = {"cachedir": cachedir,
|
conf.logdir = logdir
|
||||||
"keepcache": 0,
|
conf.cachedir = cachedir
|
||||||
"gpgcheck": 0,
|
|
||||||
"plugins": 0,
|
# Turn off logging to the console
|
||||||
"reposdir": "",
|
conf.debuglevel = 0
|
||||||
"tsflags": "nodocs"}
|
conf.errorlevel = 0
|
||||||
|
dnfbase.logging.setup_from_dnf_conf(conf)
|
||||||
|
|
||||||
|
conf.releasever = releasever
|
||||||
|
conf.installroot = installroot
|
||||||
|
conf.prepend_installroot('persistdir')
|
||||||
|
conf.tsflags.append('nodocs')
|
||||||
|
|
||||||
if proxy:
|
if proxy:
|
||||||
data["proxy"] = proxy
|
conf.proxy = proxy
|
||||||
|
|
||||||
if excludepkgs:
|
# add the repositories
|
||||||
data["exclude"] = " ".join(excludepkgs)
|
for i, r in enumerate(repositories):
|
||||||
|
repo_name = "lorax-repo-%d" % i
|
||||||
c.add_section(section)
|
repo = dnf.repo.Repo(repo_name, cachedir)
|
||||||
map(lambda (key, value): c.set(section, key, value), data.items())
|
repo.baseurl = [r]
|
||||||
|
if proxy:
|
||||||
# add the main repository - the first repository from list
|
repo.proxy = proxy
|
||||||
section = "lorax-repo"
|
repo.enable()
|
||||||
data = {"name": "lorax repo",
|
dnfbase.repos.add(repo)
|
||||||
"baseurl": repositories[0],
|
log.info("Added '%s': %s", repo_name, r)
|
||||||
"enabled": 1}
|
log.info("Fetching metadata...")
|
||||||
|
try:
|
||||||
c.add_section(section)
|
repo.load()
|
||||||
map(lambda (key, value): c.set(section, key, value), data.items())
|
except dnf.exceptions.RepoError as e:
|
||||||
|
log.error("Error fetching metadata for %s: %s", repo_name, e)
|
||||||
# add the extra repositories
|
return None
|
||||||
for n, extra in enumerate(repositories[1:], start=1):
|
|
||||||
section = "lorax-extra-repo-{0:d}".format(n)
|
|
||||||
data = {"name": "lorax extra repo {0:d}".format(n),
|
|
||||||
"baseurl": extra,
|
|
||||||
"enabled": 1}
|
|
||||||
|
|
||||||
c.add_section(section)
|
|
||||||
map(lambda (key, value): c.set(section, key, value), data.items())
|
|
||||||
|
|
||||||
# add the mirrorlists
|
# add the mirrorlists
|
||||||
for n, mirror in enumerate(mirrorlists, start=1):
|
for i, r in enumerate(mirrorlists):
|
||||||
section = "lorax-mirrorlist-{0:d}".format(n)
|
repo_name = "lorax-mirrorlist-%d" % i
|
||||||
data = {"name": "lorax mirrorlist {0:d}".format(n),
|
repo = dnf.repo.Repo(repo_name, cachedir)
|
||||||
"mirrorlist": mirror,
|
repo.mirrorlist = r
|
||||||
"enabled": 1 }
|
if proxy:
|
||||||
|
repo.proxy = proxy
|
||||||
|
repo.enable()
|
||||||
|
dnfbase.repos.add(repo)
|
||||||
|
log.info("Added '%s': %s", repo_name, r)
|
||||||
|
log.info("Fetching metadata...")
|
||||||
|
try:
|
||||||
|
repo.load()
|
||||||
|
except dnf.exceptions.RepoError as e:
|
||||||
|
log.error("Error fetching metadata for %s: %s", repo_name, e)
|
||||||
|
return None
|
||||||
|
|
||||||
c.add_section(section)
|
dnfbase.fill_sack(load_system_repo=False)
|
||||||
map(lambda (key, value): c.set(section, key, value), data.items())
|
dnfbase.read_comps()
|
||||||
|
|
||||||
# write the yum configuration file
|
return dnfbase
|
||||||
with open(yumconf, "w") as f:
|
|
||||||
c.write(f)
|
|
||||||
|
|
||||||
# create the yum base object
|
|
||||||
yb = yum.YumBase()
|
|
||||||
|
|
||||||
yb.preconf.fn = yumconf
|
|
||||||
yb.preconf.root = installroot
|
|
||||||
#yb.repos.setCacheDir(cachedir)
|
|
||||||
|
|
||||||
# Turn on as much yum logging as we can
|
|
||||||
yb.preconf.debuglevel = 6
|
|
||||||
yb.preconf.errorlevel = 6
|
|
||||||
yb.logger.setLevel(logging.DEBUG)
|
|
||||||
yb.verbose_logger.setLevel(logging.DEBUG)
|
|
||||||
|
|
||||||
return yb
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
Loading…
Reference in New Issue
Block a user