Switch lorax to use dnf instead of yum
pylorax users will need to change to using dnf and pass a dnf.Base()
object as the dbo argument instead of a yum object as the yum or ybo
argument. See the lorax script for an example of how to do this.
The lorax cmdline argument --excludepkgs has been removed since dnf
doesn't appear to have any way to support it and packages should be
controlled using templates anyway.
(cherry picked from commit 431ca6cea4)
This commit is contained in:
parent
a612c1ebd0
commit
870c4ab57f
@ -37,9 +37,9 @@ Requires: squashfs-tools >= 4.2
|
|||||||
Requires: util-linux
|
Requires: util-linux
|
||||||
Requires: xz
|
Requires: xz
|
||||||
Requires: pigz
|
Requires: pigz
|
||||||
Requires: yum
|
|
||||||
Requires: pykickstart
|
Requires: pykickstart
|
||||||
Requires: dracut >= 030
|
Requires: dracut >= 030
|
||||||
|
Requires: dnf
|
||||||
|
|
||||||
%if 0%{?fedora}
|
%if 0%{?fedora}
|
||||||
# Fedora specific deps
|
# Fedora specific deps
|
||||||
|
|||||||
@ -36,7 +36,7 @@ import selinux
|
|||||||
from pylorax.base import BaseLoraxClass, DataHolder
|
from pylorax.base import BaseLoraxClass, DataHolder
|
||||||
import pylorax.output as output
|
import pylorax.output as output
|
||||||
|
|
||||||
import yum
|
import dnf
|
||||||
|
|
||||||
from pylorax.sysutils import joinpaths, remove, linktree
|
from pylorax.sysutils import joinpaths, remove, linktree
|
||||||
from rpmUtils.arch import getBaseArch
|
from rpmUtils.arch import getBaseArch
|
||||||
@ -96,9 +96,6 @@ class Lorax(BaseLoraxClass):
|
|||||||
self.conf.add_section("templates")
|
self.conf.add_section("templates")
|
||||||
self.conf.set("templates", "ramdisk", "ramdisk.ltmpl")
|
self.conf.set("templates", "ramdisk", "ramdisk.ltmpl")
|
||||||
|
|
||||||
self.conf.add_section("yum")
|
|
||||||
self.conf.set("yum", "skipbroken", "0")
|
|
||||||
|
|
||||||
self.conf.add_section("compression")
|
self.conf.add_section("compression")
|
||||||
self.conf.set("compression", "type", "xz")
|
self.conf.set("compression", "type", "xz")
|
||||||
self.conf.set("compression", "args", "")
|
self.conf.set("compression", "args", "")
|
||||||
@ -149,7 +146,7 @@ class Lorax(BaseLoraxClass):
|
|||||||
fh.setLevel(logging.DEBUG)
|
fh.setLevel(logging.DEBUG)
|
||||||
logger.addHandler(fh)
|
logger.addHandler(fh)
|
||||||
|
|
||||||
def run(self, ybo, product, version, release, variant="", bugurl="",
|
def run(self, dbo, product, version, release, variant="", bugurl="",
|
||||||
isfinal=False, workdir=None, outputdir=None, buildarch=None, volid=None,
|
isfinal=False, workdir=None, outputdir=None, buildarch=None, volid=None,
|
||||||
domacboot=True, doupgrade=True, remove_temp=False,
|
domacboot=True, doupgrade=True, remove_temp=False,
|
||||||
installpkgs=None,
|
installpkgs=None,
|
||||||
@ -221,16 +218,16 @@ class Lorax(BaseLoraxClass):
|
|||||||
logger.critical("selinux must be disabled or in Permissive mode")
|
logger.critical("selinux must be disabled or in Permissive mode")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
# do we have a proper yum base object?
|
# do we have a proper dnf base object?
|
||||||
logger.info("checking yum base object")
|
logger.info("checking dnf base object")
|
||||||
if not isinstance(ybo, yum.YumBase):
|
if not isinstance(dbo, dnf.Base):
|
||||||
logger.critical("no yum base object")
|
logger.critical("no dnf base object")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
self.inroot = ybo.conf.installroot
|
self.inroot = dbo.conf.installroot
|
||||||
logger.debug("using install root: {0}".format(self.inroot))
|
logger.debug("using install root: {0}".format(self.inroot))
|
||||||
|
|
||||||
if not buildarch:
|
if not buildarch:
|
||||||
buildarch = get_buildarch(ybo)
|
buildarch = get_buildarch(dbo)
|
||||||
|
|
||||||
logger.info("setting up build architecture")
|
logger.info("setting up build architecture")
|
||||||
self.arch = ArchData(buildarch)
|
self.arch = ArchData(buildarch)
|
||||||
@ -253,15 +250,14 @@ class Lorax(BaseLoraxClass):
|
|||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
templatedir = self.conf.get("lorax", "sharedir")
|
templatedir = self.conf.get("lorax", "sharedir")
|
||||||
# NOTE: rb.root = ybo.conf.installroot (== self.inroot)
|
# NOTE: rb.root = dbo.conf.installroot (== self.inroot)
|
||||||
rb = RuntimeBuilder(product=self.product, arch=self.arch,
|
rb = RuntimeBuilder(product=self.product, arch=self.arch,
|
||||||
yum=ybo, templatedir=templatedir,
|
dbo=dbo, templatedir=templatedir,
|
||||||
installpkgs=installpkgs,
|
installpkgs=installpkgs,
|
||||||
add_templates=add_templates,
|
add_templates=add_templates,
|
||||||
add_template_vars=add_template_vars)
|
add_template_vars=add_template_vars)
|
||||||
|
|
||||||
logger.info("installing runtime packages")
|
logger.info("installing runtime packages")
|
||||||
rb.yum.conf.skip_broken = self.conf.getboolean("yum", "skipbroken")
|
|
||||||
rb.install()
|
rb.install()
|
||||||
|
|
||||||
# write .buildstamp
|
# write .buildstamp
|
||||||
@ -358,10 +354,12 @@ class Lorax(BaseLoraxClass):
|
|||||||
remove(self.workdir)
|
remove(self.workdir)
|
||||||
|
|
||||||
|
|
||||||
def get_buildarch(ybo):
|
def get_buildarch(dbo):
|
||||||
# get architecture of the available anaconda package
|
# get architecture of the available anaconda package
|
||||||
buildarch = None
|
buildarch = None
|
||||||
for anaconda in ybo.doPackageLists(patterns=["anaconda"]).available:
|
q = dbo.sack.query()
|
||||||
|
a = q.available()
|
||||||
|
for anaconda in a.filter(name="anaconda"):
|
||||||
if anaconda.arch != "src":
|
if anaconda.arch != "src":
|
||||||
buildarch = anaconda.arch
|
buildarch = anaconda.arch
|
||||||
break
|
break
|
||||||
|
|||||||
107
src/pylorax/dnfhelper.py
Normal file
107
src/pylorax/dnfhelper.py
Normal file
@ -0,0 +1,107 @@
|
|||||||
|
#
|
||||||
|
# dnfhelper.py
|
||||||
|
#
|
||||||
|
# Copyright (C) 2010-2014 Red Hat, Inc.
|
||||||
|
#
|
||||||
|
# This program is free software; you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU General Public License as published by
|
||||||
|
# the Free Software Foundation; either version 2 of the License, or
|
||||||
|
# (at your option) any later version.
|
||||||
|
#
|
||||||
|
# This program is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU General Public License
|
||||||
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
#
|
||||||
|
# Red Hat Author(s): Martin Gracik <mgracik@redhat.com>
|
||||||
|
# Brian C. Lane <bcl@redhat.com>
|
||||||
|
#
|
||||||
|
|
||||||
|
import logging
|
||||||
|
logger = logging.getLogger("pylorax.dnfhelper")
|
||||||
|
import dnf
|
||||||
|
import collections
|
||||||
|
import time
|
||||||
|
import pylorax.output as output
|
||||||
|
|
||||||
|
__all__ = ['LoraxDownloadCallback', 'LoraxRpmCallback']
|
||||||
|
|
||||||
|
def _paced(fn):
|
||||||
|
"""Execute `fn` no more often then every 2 seconds."""
|
||||||
|
def paced_fn(self, *args):
|
||||||
|
now = time.time()
|
||||||
|
if now - self.last_time < 2:
|
||||||
|
return
|
||||||
|
self.last_time = now
|
||||||
|
return fn(self, *args)
|
||||||
|
return paced_fn
|
||||||
|
|
||||||
|
|
||||||
|
class LoraxDownloadCallback(dnf.callback.DownloadProgress):
|
||||||
|
def __init__(self):
|
||||||
|
self.downloads = collections.defaultdict(int)
|
||||||
|
self.last_time = time.time()
|
||||||
|
self.total_files = 0
|
||||||
|
self.total_size = 0
|
||||||
|
|
||||||
|
self.pkgno = 0
|
||||||
|
self.total = 0
|
||||||
|
|
||||||
|
self.output = output.LoraxOutput()
|
||||||
|
|
||||||
|
@_paced
|
||||||
|
def _update(self):
|
||||||
|
msg = "Downloading %(pkgno)s / %(total_files)s RPMs, " \
|
||||||
|
"%(downloaded)s / %(total_size)s (%(percent)d%%) done.\n"
|
||||||
|
downloaded = sum(self.downloads.values())
|
||||||
|
vals = {
|
||||||
|
'downloaded' : downloaded,
|
||||||
|
'percent' : int(100 * downloaded/self.total_size),
|
||||||
|
'pkgno' : self.pkgno,
|
||||||
|
'total_files' : self.total_files,
|
||||||
|
'total_size' : self.total_size
|
||||||
|
}
|
||||||
|
self.output.write(msg % vals)
|
||||||
|
|
||||||
|
def end(self, payload, status, err_msg):
|
||||||
|
nevra = str(payload)
|
||||||
|
if status is dnf.callback.STATUS_OK:
|
||||||
|
self.downloads[nevra] = payload.download_size
|
||||||
|
self.pkgno += 1
|
||||||
|
self._update()
|
||||||
|
return
|
||||||
|
logger.critical("Failed to download '%s': %d - %s", nevra, status, err_msg)
|
||||||
|
|
||||||
|
def progress(self, payload, done):
|
||||||
|
nevra = str(payload)
|
||||||
|
self.downloads[nevra] = done
|
||||||
|
self._update()
|
||||||
|
|
||||||
|
def start(self, total_files, total_size):
|
||||||
|
self.total_files = total_files
|
||||||
|
self.total_size = total_size
|
||||||
|
|
||||||
|
|
||||||
|
class LoraxRpmCallback(dnf.callback.LoggingTransactionDisplay):
|
||||||
|
def __init__(self, queue):
|
||||||
|
super(LoraxRpmCallback, self).__init__()
|
||||||
|
self._queue = queue
|
||||||
|
self._last_ts = None
|
||||||
|
self.cnt = 0
|
||||||
|
|
||||||
|
def event(self, package, action, te_current, te_total, ts_current, ts_total):
|
||||||
|
if action == self.PKG_INSTALL and te_current == 0:
|
||||||
|
# do not report same package twice
|
||||||
|
if self._last_ts == ts_current:
|
||||||
|
return
|
||||||
|
self._last_ts = ts_current
|
||||||
|
|
||||||
|
msg = '(%d/%d) %s.%s' % \
|
||||||
|
(ts_current, ts_total, package.name, package.arch)
|
||||||
|
self.cnt += 1
|
||||||
|
self._queue.put(('install', msg))
|
||||||
|
elif action == self.TRANS_POST:
|
||||||
|
self._queue.put(('post', None))
|
||||||
@ -28,15 +28,18 @@ from os.path import basename, isdir
|
|||||||
from subprocess import CalledProcessError
|
from subprocess import CalledProcessError
|
||||||
|
|
||||||
from pylorax.sysutils import joinpaths, cpfile, mvfile, replace, remove
|
from pylorax.sysutils import joinpaths, cpfile, mvfile, replace, remove
|
||||||
from pylorax.yumhelper import LoraxDownloadCallback, LoraxTransactionCallback, LoraxRpmCallback
|
from pylorax.dnfhelper import LoraxDownloadCallback, LoraxRpmCallback
|
||||||
from pylorax.base import DataHolder
|
from pylorax.base import DataHolder
|
||||||
from pylorax.executils import runcmd, runcmd_output
|
from pylorax.executils import runcmd, runcmd_output
|
||||||
from pylorax.imgutils import mkcpio
|
from pylorax.imgutils import mkcpio
|
||||||
|
import pylorax.output as output
|
||||||
|
|
||||||
from mako.lookup import TemplateLookup
|
from mako.lookup import TemplateLookup
|
||||||
from mako.exceptions import text_error_template
|
from mako.exceptions import text_error_template
|
||||||
import sys, traceback
|
import sys, traceback
|
||||||
import struct
|
import struct
|
||||||
|
import dnf
|
||||||
|
import multiprocessing
|
||||||
|
|
||||||
class LoraxTemplate(object):
|
class LoraxTemplate(object):
|
||||||
def __init__(self, directories=None):
|
def __init__(self, directories=None):
|
||||||
@ -108,7 +111,7 @@ class LoraxTemplateRunner(object):
|
|||||||
This class parses and executes Lorax templates. Sample usage:
|
This class parses and executes Lorax templates. Sample usage:
|
||||||
|
|
||||||
# install a bunch of packages
|
# install a bunch of packages
|
||||||
runner = LoraxTemplateRunner(inroot=rundir, outroot=rundir, yum=yum_obj)
|
runner = LoraxTemplateRunner(inroot=rundir, outroot=rundir, dbo=dnf_obj)
|
||||||
runner.run("install-packages.ltmpl")
|
runner.run("install-packages.ltmpl")
|
||||||
|
|
||||||
# modify a runtime dir
|
# modify a runtime dir
|
||||||
@ -145,11 +148,11 @@ class LoraxTemplateRunner(object):
|
|||||||
|
|
||||||
* Commands should raise exceptions for errors - don't use sys.exit()
|
* Commands should raise exceptions for errors - don't use sys.exit()
|
||||||
'''
|
'''
|
||||||
def __init__(self, inroot, outroot, yum=None, fatalerrors=True,
|
def __init__(self, inroot, outroot, dbo=None, fatalerrors=True,
|
||||||
templatedir=None, defaults=None):
|
templatedir=None, defaults=None):
|
||||||
self.inroot = inroot
|
self.inroot = inroot
|
||||||
self.outroot = outroot
|
self.outroot = outroot
|
||||||
self.yum = yum
|
self.dbo = dbo
|
||||||
self.fatalerrors = fatalerrors
|
self.fatalerrors = fatalerrors
|
||||||
self.templatedir = templatedir or "/usr/share/lorax"
|
self.templatedir = templatedir or "/usr/share/lorax"
|
||||||
self.templatefile = None
|
self.templatefile = None
|
||||||
@ -166,8 +169,8 @@ class LoraxTemplateRunner(object):
|
|||||||
return joinpaths(self.inroot, path)
|
return joinpaths(self.inroot, path)
|
||||||
|
|
||||||
def _filelist(self, *pkgs):
|
def _filelist(self, *pkgs):
|
||||||
pkglist = self.yum.doPackageLists(pkgnarrow="installed", patterns=pkgs)
|
pkglist = self.dbo.doPackageLists(pkgnarrow="installed", patterns=pkgs)
|
||||||
return set([f for pkg in pkglist.installed for f in pkg.filelist+pkg.ghostlist])
|
return set([f for pkg in pkglist for f in pkg.files])
|
||||||
|
|
||||||
def _getsize(self, *files):
|
def _getsize(self, *files):
|
||||||
return sum(os.path.getsize(self._out(f)) for f in files if os.path.isfile(self._out(f)))
|
return sum(os.path.getsize(self._out(f)) for f in files if os.path.isfile(self._out(f)))
|
||||||
@ -447,9 +450,9 @@ class LoraxTemplateRunner(object):
|
|||||||
cmd = cmd[1:]
|
cmd = cmd[1:]
|
||||||
|
|
||||||
try:
|
try:
|
||||||
output = runcmd_output(cmd, cwd=cwd)
|
stdout = runcmd_output(cmd, cwd=cwd)
|
||||||
if output:
|
if stdout:
|
||||||
logger.debug('command output:\n%s', output)
|
logger.debug('command output:\n%s', stdout)
|
||||||
logger.debug("command finished successfully")
|
logger.debug("command finished successfully")
|
||||||
except CalledProcessError as e:
|
except CalledProcessError as e:
|
||||||
if e.output:
|
if e.output:
|
||||||
@ -471,7 +474,7 @@ class LoraxTemplateRunner(object):
|
|||||||
|
|
||||||
for p in pkgs:
|
for p in pkgs:
|
||||||
try:
|
try:
|
||||||
self.yum.install(pattern=p)
|
self.dbo.install(p)
|
||||||
except Exception as e: # pylint: disable=broad-except
|
except Exception as e: # pylint: disable=broad-except
|
||||||
# FIXME: save exception and re-raise after the loop finishes
|
# FIXME: save exception and re-raise after the loop finishes
|
||||||
logger.error("installpkg %s failed: %s", p, str(e))
|
logger.error("installpkg %s failed: %s", p, str(e))
|
||||||
@ -501,19 +504,60 @@ class LoraxTemplateRunner(object):
|
|||||||
Actually install all the packages requested by previous 'installpkg'
|
Actually install all the packages requested by previous 'installpkg'
|
||||||
commands.
|
commands.
|
||||||
'''
|
'''
|
||||||
self.yum.buildTransaction()
|
|
||||||
dl_callback = LoraxDownloadCallback()
|
def do_transaction(base, queue):
|
||||||
self.yum.repos.setProgressBar(dl_callback)
|
try:
|
||||||
self.yum.processTransaction(callback=LoraxTransactionCallback(dl_callback),
|
display = LoraxRpmCallback(queue)
|
||||||
rpmDisplay=LoraxRpmCallback())
|
base.do_transaction(display=display)
|
||||||
|
except BaseException as e:
|
||||||
|
logger.error("The transaction process has ended abruptly: %s", e)
|
||||||
|
queue.put(('quit', str(e)))
|
||||||
|
|
||||||
|
self.dbo.reset()
|
||||||
|
try:
|
||||||
|
logger.info("Checking dependencies")
|
||||||
|
self.dbo.resolve()
|
||||||
|
except dnf.exceptions.DepsolveError as e:
|
||||||
|
logger.error("Dependency check failed: %s", e)
|
||||||
|
raise
|
||||||
|
logger.info("%d packages selected", len(self.dbo.transaction))
|
||||||
|
if len(self.dbo.transaction) == 0:
|
||||||
|
raise Exception("No packages in transaction")
|
||||||
|
|
||||||
|
pkgs_to_download = self.dbo.transaction.install_set
|
||||||
|
logger.info("Downloading packages")
|
||||||
|
progress = LoraxDownloadCallback()
|
||||||
|
try:
|
||||||
|
self.dbo.download_packages(pkgs_to_download, progress)
|
||||||
|
except dnf.exceptions.DownloadError as e:
|
||||||
|
logger.error("Failed to download the following packages: %s", e)
|
||||||
|
raise
|
||||||
|
|
||||||
|
logger.info("Preparing transaction from installation source")
|
||||||
|
queue = multiprocessing.Queue()
|
||||||
|
msgout = output.LoraxOutput()
|
||||||
|
process = multiprocessing.Process(target=do_transaction, args=(self.dbo, queue))
|
||||||
|
process.start()
|
||||||
|
(token, msg) = queue.get()
|
||||||
|
while token not in ('post', 'quit'):
|
||||||
|
if token == 'install':
|
||||||
|
logging.info("%s", msg)
|
||||||
|
msgout.writeline(msg)
|
||||||
|
(token, msg) = queue.get()
|
||||||
|
if token == 'quit':
|
||||||
|
logger.error("Transaction failed.")
|
||||||
|
raise Exception("Transaction failed")
|
||||||
|
|
||||||
|
logger.info("Performing post-installation setup tasks")
|
||||||
|
process.join()
|
||||||
|
|
||||||
# verify if all packages that were supposed to be installed,
|
# verify if all packages that were supposed to be installed,
|
||||||
# are really installed
|
# are really installed
|
||||||
errs = [t.po for t in self.yum.tsInfo if not self.yum.rpmdb.contains(po=t.po)]
|
errs = [t.po for t in self.dbo.tsInfo if not self.dbo.rpmdb.contains(po=t.po)]
|
||||||
for po in errs:
|
for po in errs:
|
||||||
logger.error("package '%s' was not installed", po)
|
logger.error("package '%s' was not installed", po)
|
||||||
|
|
||||||
self.yum.closeRpmDB()
|
self.dbo.close()
|
||||||
|
|
||||||
def removefrom(self, pkg, *globs):
|
def removefrom(self, pkg, *globs):
|
||||||
'''
|
'''
|
||||||
|
|||||||
@ -67,19 +67,19 @@ def generate_module_info(moddir, outfile=None):
|
|||||||
|
|
||||||
class RuntimeBuilder(object):
|
class RuntimeBuilder(object):
|
||||||
'''Builds the anaconda runtime image.'''
|
'''Builds the anaconda runtime image.'''
|
||||||
def __init__(self, product, arch, yum, templatedir=None,
|
def __init__(self, product, arch, dbo, templatedir=None,
|
||||||
installpkgs=None,
|
installpkgs=None,
|
||||||
add_templates=None,
|
add_templates=None,
|
||||||
add_template_vars=None):
|
add_template_vars=None):
|
||||||
root = yum.conf.installroot
|
root = dbo.conf.installroot
|
||||||
# use a copy of product so we can modify it locally
|
# use a copy of product so we can modify it locally
|
||||||
product = product.copy()
|
product = product.copy()
|
||||||
product.name = product.name.lower()
|
product.name = product.name.lower()
|
||||||
self.vars = DataHolder(arch=arch, product=product, yum=yum, root=root,
|
self.vars = DataHolder(arch=arch, product=product, dbo=dbo, root=root,
|
||||||
basearch=arch.basearch, libdir=arch.libdir)
|
basearch=arch.basearch, libdir=arch.libdir)
|
||||||
self.yum = yum
|
self.dbo = dbo
|
||||||
self._runner = LoraxTemplateRunner(inroot=root, outroot=root,
|
self._runner = LoraxTemplateRunner(inroot=root, outroot=root,
|
||||||
yum=yum, templatedir=templatedir)
|
dbo=dbo, templatedir=templatedir)
|
||||||
self.add_templates = add_templates or []
|
self.add_templates = add_templates or []
|
||||||
self.add_template_vars = add_template_vars or {}
|
self.add_template_vars = add_template_vars or {}
|
||||||
self._installpkgs = installpkgs or []
|
self._installpkgs = installpkgs or []
|
||||||
@ -87,7 +87,9 @@ class RuntimeBuilder(object):
|
|||||||
|
|
||||||
def _install_branding(self):
|
def _install_branding(self):
|
||||||
release = None
|
release = None
|
||||||
for pkg in self.yum.whatProvides('/etc/system-release', None, None):
|
q = self.dbo.sack.query()
|
||||||
|
a = q.available()
|
||||||
|
for pkg in a.filter(provides='/etc/system-release'):
|
||||||
if pkg.name.startswith('generic'):
|
if pkg.name.startswith('generic'):
|
||||||
continue
|
continue
|
||||||
else:
|
else:
|
||||||
@ -119,9 +121,10 @@ class RuntimeBuilder(object):
|
|||||||
'''debugging data: write out lists of package contents'''
|
'''debugging data: write out lists of package contents'''
|
||||||
if not os.path.isdir(pkglistdir):
|
if not os.path.isdir(pkglistdir):
|
||||||
os.makedirs(pkglistdir)
|
os.makedirs(pkglistdir)
|
||||||
for pkgobj in self.yum.doPackageLists(pkgnarrow='installed').installed:
|
q = self.dbo.sack.query()
|
||||||
|
for pkgobj in q.installed():
|
||||||
with open(joinpaths(pkglistdir, pkgobj.name), "w") as fobj:
|
with open(joinpaths(pkglistdir, pkgobj.name), "w") as fobj:
|
||||||
for fname in pkgobj.filelist + pkgobj.dirlist:
|
for fname in pkgobj.files:
|
||||||
fobj.write("{0}\n".format(fname))
|
fobj.write("{0}\n".format(fname))
|
||||||
|
|
||||||
def postinstall(self):
|
def postinstall(self):
|
||||||
@ -143,8 +146,9 @@ class RuntimeBuilder(object):
|
|||||||
'''debugging data: write a big list of pkg sizes'''
|
'''debugging data: write a big list of pkg sizes'''
|
||||||
fobj = open(pkgsizefile, "w")
|
fobj = open(pkgsizefile, "w")
|
||||||
getsize = lambda f: os.lstat(f).st_size if os.path.exists(f) else 0
|
getsize = lambda f: os.lstat(f).st_size if os.path.exists(f) else 0
|
||||||
for p in sorted(self.yum.doPackageLists(pkgnarrow='installed').installed):
|
q = self.dbo.sack.query()
|
||||||
pkgsize = sum(getsize(joinpaths(self.vars.root,f)) for f in p.filelist)
|
for p in sorted(q.installed()):
|
||||||
|
pkgsize = sum(getsize(joinpaths(self.vars.root,f)) for f in p.files)
|
||||||
fobj.write("{0.name}.{0.arch}: {1}\n".format(p, pkgsize))
|
fobj.write("{0.name}.{0.arch}: {1}\n".format(p, pkgsize))
|
||||||
|
|
||||||
def generate_module_data(self):
|
def generate_module_data(self):
|
||||||
|
|||||||
@ -1,127 +0,0 @@
|
|||||||
#
|
|
||||||
# yumhelper.py
|
|
||||||
#
|
|
||||||
# Copyright (C) 2010-2014 Red Hat, Inc.
|
|
||||||
#
|
|
||||||
# This program is free software; you can redistribute it and/or modify
|
|
||||||
# it under the terms of the GNU General Public License as published by
|
|
||||||
# the Free Software Foundation; either version 2 of the License, or
|
|
||||||
# (at your option) any later version.
|
|
||||||
#
|
|
||||||
# This program is distributed in the hope that it will be useful,
|
|
||||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
# GNU General Public License for more details.
|
|
||||||
#
|
|
||||||
# You should have received a copy of the GNU General Public License
|
|
||||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
#
|
|
||||||
# Red Hat Author(s): Martin Gracik <mgracik@redhat.com>
|
|
||||||
#
|
|
||||||
|
|
||||||
import logging
|
|
||||||
logger = logging.getLogger("pylorax.yumhelper")
|
|
||||||
import sys
|
|
||||||
import yum, yum.callbacks, yum.rpmtrans
|
|
||||||
import pylorax.output as output
|
|
||||||
|
|
||||||
__all__ = ['LoraxDownloadCallback', 'LoraxTransactionCallback',
|
|
||||||
'LoraxRpmCallback']
|
|
||||||
|
|
||||||
class LoraxDownloadCallback(yum.callbacks.DownloadBaseCallback):
|
|
||||||
def __init__(self):
|
|
||||||
yum.callbacks.DownloadBaseCallback.__init__(self)
|
|
||||||
|
|
||||||
self.pkgno = 0
|
|
||||||
self.total = 0
|
|
||||||
|
|
||||||
self.output = output.LoraxOutput()
|
|
||||||
|
|
||||||
|
|
||||||
def updateProgress(self, name, frac, fread, ftime):
|
|
||||||
"""
|
|
||||||
Update the progress bar
|
|
||||||
@param name: filename
|
|
||||||
@param frac: progress fraction (0 -> 1)
|
|
||||||
@param fread: formated string containing BytesRead
|
|
||||||
@param ftime: formated string containing remaining or elapsed time
|
|
||||||
"""
|
|
||||||
# Only update when it is finished downloading
|
|
||||||
if frac < 1:
|
|
||||||
return
|
|
||||||
|
|
||||||
self.pkgno += 1
|
|
||||||
info = "({0:3d}/{1:3d}) "
|
|
||||||
info = info.format(self.pkgno, self.total)
|
|
||||||
|
|
||||||
infolen, pkglen = len(info), len(name)
|
|
||||||
if (infolen + pkglen) > self.output.width:
|
|
||||||
name = "{0}...".format(name[:self.output.width-infolen-3])
|
|
||||||
|
|
||||||
msg = "{0}<b>{1}</b>\n".format(info, name)
|
|
||||||
self.output.write(msg)
|
|
||||||
|
|
||||||
|
|
||||||
class LoraxTransactionCallback(object):
|
|
||||||
|
|
||||||
def __init__(self, dl_callback):
|
|
||||||
self.output = output.LoraxOutput()
|
|
||||||
|
|
||||||
self.dl_callback = dl_callback
|
|
||||||
|
|
||||||
def event(self, state, data=None):
|
|
||||||
if state == yum.callbacks.PT_DOWNLOAD:
|
|
||||||
self.output.write("downloading packages\n")
|
|
||||||
elif state == yum.callbacks.PT_DOWNLOAD_PKGS:
|
|
||||||
# Initialize the total number of packages being downloaded
|
|
||||||
self.dl_callback.total = len(data)
|
|
||||||
elif state == yum.callbacks.PT_GPGCHECK:
|
|
||||||
self.output.write("checking package signatures\n")
|
|
||||||
elif state == yum.callbacks.PT_TEST_TRANS:
|
|
||||||
self.output.write("running test transaction\n")
|
|
||||||
elif state == yum.callbacks.PT_TRANSACTION:
|
|
||||||
self.output.write("running transaction\n")
|
|
||||||
|
|
||||||
|
|
||||||
class LoraxRpmCallback(yum.rpmtrans.RPMBaseCallback):
|
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
yum.rpmtrans.RPMBaseCallback.__init__(self)
|
|
||||||
self.output = output.LoraxOutput()
|
|
||||||
|
|
||||||
def event(self, package, action, te_current, te_total,
|
|
||||||
ts_current, ts_total):
|
|
||||||
|
|
||||||
action_str = self.action[action].encode("utf-8")
|
|
||||||
info = "({0:3d}/{1:3d}) [{2:3.0f}%] {3} "
|
|
||||||
info = info.format(ts_current, ts_total,
|
|
||||||
float(te_current) / float(te_total) * 100,
|
|
||||||
action_str.lower())
|
|
||||||
|
|
||||||
pkg = "{0}".format(package)
|
|
||||||
|
|
||||||
infolen, pkglen = len(info), len(pkg)
|
|
||||||
if (infolen + pkglen) > self.output.width:
|
|
||||||
pkg = "{0}...".format(pkg[:self.output.width-infolen-3])
|
|
||||||
|
|
||||||
msg = "{0}<b>{1}</b>".format(info, pkg)
|
|
||||||
|
|
||||||
# When not outputting to a tty we only want to print it once at the end
|
|
||||||
if sys.stdout.isatty():
|
|
||||||
self.output.write(msg + "\r")
|
|
||||||
if te_current == te_total:
|
|
||||||
self.output.write("\n")
|
|
||||||
elif te_current == te_total:
|
|
||||||
self.output.write(msg + "\n")
|
|
||||||
|
|
||||||
|
|
||||||
def filelog(self, package, action):
|
|
||||||
if self.fileaction.get(action) == "Installed":
|
|
||||||
logger.debug("{0} installed successfully".format(package))
|
|
||||||
|
|
||||||
def errorlog(self, msg):
|
|
||||||
logger.warning("RPM transaction error: %s", msg)
|
|
||||||
|
|
||||||
def scriptout(self, package, msgs):
|
|
||||||
if msgs:
|
|
||||||
logger.info("%s scriptlet output:\n%s", package, msgs)
|
|
||||||
161
src/sbin/lorax
161
src/sbin/lorax
@ -25,20 +25,16 @@ import logging
|
|||||||
log = logging.getLogger("lorax")
|
log = logging.getLogger("lorax")
|
||||||
program_log = logging.getLogger("program")
|
program_log = logging.getLogger("program")
|
||||||
pylorax_log = logging.getLogger("pylorax")
|
pylorax_log = logging.getLogger("pylorax")
|
||||||
yum_log = logging.getLogger("yum")
|
dnf_log = logging.getLogger("dnf")
|
||||||
|
|
||||||
|
|
||||||
import sys
|
import sys
|
||||||
import os
|
import os
|
||||||
import tempfile
|
import tempfile
|
||||||
from optparse import OptionParser, OptionGroup
|
from optparse import OptionParser, OptionGroup
|
||||||
import ConfigParser
|
|
||||||
import shutil
|
import shutil
|
||||||
|
|
||||||
import yum
|
import dnf
|
||||||
# This is a bit of a hack to short circuit yum's internal logging
|
|
||||||
# handler setup. We already set one up so we don't need it to run.
|
|
||||||
yum.logginglevels._added_handlers = True
|
|
||||||
import pylorax
|
import pylorax
|
||||||
|
|
||||||
def setup_logging(opts):
|
def setup_logging(opts):
|
||||||
@ -67,12 +63,12 @@ def setup_logging(opts):
|
|||||||
fh.setLevel(logging.DEBUG)
|
fh.setLevel(logging.DEBUG)
|
||||||
program_log.addHandler(fh)
|
program_log.addHandler(fh)
|
||||||
|
|
||||||
# yum logging
|
# dnf logging
|
||||||
yum_log.setLevel(logging.DEBUG)
|
dnf_log.setLevel(logging.DEBUG)
|
||||||
logfile = os.path.abspath(os.path.dirname(opts.logfile))+"/yum.log"
|
logfile = os.path.abspath(os.path.dirname(opts.logfile))+"/dnf.log"
|
||||||
fh = logging.FileHandler(filename=logfile, mode="w")
|
fh = logging.FileHandler(filename=logfile, mode="w")
|
||||||
fh.setLevel(logging.DEBUG)
|
fh.setLevel(logging.DEBUG)
|
||||||
yum_log.addHandler(fh)
|
dnf_log.addHandler(fh)
|
||||||
|
|
||||||
|
|
||||||
def main(args):
|
def main(args):
|
||||||
@ -116,9 +112,6 @@ def main(args):
|
|||||||
help="config file", metavar="STRING")
|
help="config file", metavar="STRING")
|
||||||
optional.add_option("--proxy", default=None,
|
optional.add_option("--proxy", default=None,
|
||||||
help="repo proxy url:port", metavar="STRING")
|
help="repo proxy url:port", metavar="STRING")
|
||||||
optional.add_option("-e", "--excludepkgs", default=[],
|
|
||||||
action="append", metavar="STRING",
|
|
||||||
help="package glob to exclude (may be listed multiple times)")
|
|
||||||
optional.add_option("-i", "--installpkgs", default=[],
|
optional.add_option("-i", "--installpkgs", default=[],
|
||||||
action="append", metavar="STRING",
|
action="append", metavar="STRING",
|
||||||
help="package glob to install before runtime-install.tmpl runs. (may be listed multiple times)")
|
help="package glob to install before runtime-install.tmpl runs. (may be listed multiple times)")
|
||||||
@ -180,17 +173,16 @@ def main(args):
|
|||||||
# create the temporary directory for lorax
|
# create the temporary directory for lorax
|
||||||
tempdir = tempfile.mkdtemp(prefix="lorax.", dir=tempfile.gettempdir())
|
tempdir = tempfile.mkdtemp(prefix="lorax.", dir=tempfile.gettempdir())
|
||||||
|
|
||||||
# create the yumbase object
|
|
||||||
installtree = os.path.join(tempdir, "installtree")
|
installtree = os.path.join(tempdir, "installtree")
|
||||||
os.mkdir(installtree)
|
os.mkdir(installtree)
|
||||||
yumtempdir = os.path.join(tempdir, "yum")
|
dnftempdir = os.path.join(tempdir, "dnf")
|
||||||
os.mkdir(yumtempdir)
|
os.mkdir(dnftempdir)
|
||||||
|
|
||||||
yb = get_yum_base_object(installtree, opts.source, opts.mirrorlist,
|
dnfbase = get_dnf_base_object(installtree, opts.source, opts.mirrorlist,
|
||||||
yumtempdir, opts.proxy, opts.excludepkgs)
|
dnftempdir, opts.proxy, opts.version)
|
||||||
|
|
||||||
if yb is None:
|
if dnfbase is None:
|
||||||
print("error: unable to create the yumbase object", file=sys.stderr)
|
print("error: unable to create the dnf base object", file=sys.stderr)
|
||||||
shutil.rmtree(tempdir)
|
shutil.rmtree(tempdir)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
@ -204,7 +196,7 @@ def main(args):
|
|||||||
# run lorax
|
# run lorax
|
||||||
lorax = pylorax.Lorax()
|
lorax = pylorax.Lorax()
|
||||||
lorax.configure(conf_file=opts.config)
|
lorax.configure(conf_file=opts.config)
|
||||||
lorax.run(yb, opts.product, opts.version, opts.release,
|
lorax.run(dnfbase, opts.product, opts.version, opts.release,
|
||||||
opts.variant, opts.bugurl, opts.isfinal,
|
opts.variant, opts.bugurl, opts.isfinal,
|
||||||
workdir=tempdir, outputdir=outputdir, buildarch=opts.buildarch,
|
workdir=tempdir, outputdir=outputdir, buildarch=opts.buildarch,
|
||||||
volid=opts.volid, domacboot=opts.domacboot, doupgrade=opts.doupgrade,
|
volid=opts.volid, domacboot=opts.domacboot, doupgrade=opts.doupgrade,
|
||||||
@ -214,9 +206,17 @@ def main(args):
|
|||||||
remove_temp=True)
|
remove_temp=True)
|
||||||
|
|
||||||
|
|
||||||
def get_yum_base_object(installroot, repositories, mirrorlists=None,
|
def get_dnf_base_object(installroot, repositories, mirrorlists=None,
|
||||||
tempdir="/var/tmp", proxy=None, excludepkgs=None):
|
tempdir="/var/tmp", proxy=None, releasever="21"):
|
||||||
|
""" Create a dnf Base object and setup the repositories and installroot
|
||||||
|
|
||||||
|
:param string installroot: Full path to the installroot
|
||||||
|
:param list repositories: List of repositories to use for the installation
|
||||||
|
:param list mirrorlist: List of mirrors to use
|
||||||
|
:param string tempdir: Path of temporary directory
|
||||||
|
:param string proxy: http proxy to use when fetching packages
|
||||||
|
:param string releasever: Release version to pass to dnf
|
||||||
|
"""
|
||||||
def sanitize_repo(repo):
|
def sanitize_repo(repo):
|
||||||
"""Convert bare paths to file:/// URIs, and silently reject protocols unhandled by yum"""
|
"""Convert bare paths to file:/// URIs, and silently reject protocols unhandled by yum"""
|
||||||
if repo.startswith("/"):
|
if repo.startswith("/"):
|
||||||
@ -227,7 +227,6 @@ def get_yum_base_object(installroot, repositories, mirrorlists=None,
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
mirrorlists = mirrorlists or []
|
mirrorlists = mirrorlists or []
|
||||||
excludepkgs = excludepkgs or []
|
|
||||||
|
|
||||||
# sanitize the repositories
|
# sanitize the repositories
|
||||||
repositories = map(sanitize_repo, repositories)
|
repositories = map(sanitize_repo, repositories)
|
||||||
@ -237,78 +236,70 @@ def get_yum_base_object(installroot, repositories, mirrorlists=None,
|
|||||||
repositories = filter(bool, repositories)
|
repositories = filter(bool, repositories)
|
||||||
mirrorlists = filter(bool, mirrorlists)
|
mirrorlists = filter(bool, mirrorlists)
|
||||||
|
|
||||||
cachedir = os.path.join(tempdir, "yum.cache")
|
cachedir = os.path.join(tempdir, "dnf.cache")
|
||||||
if not os.path.isdir(cachedir):
|
if not os.path.isdir(cachedir):
|
||||||
os.mkdir(cachedir)
|
os.mkdir(cachedir)
|
||||||
|
|
||||||
yumconf = os.path.join(tempdir, "yum.conf")
|
logdir = os.path.join(tempdir, "dnf.logs")
|
||||||
c = ConfigParser.ConfigParser()
|
if not os.path.isdir(logdir):
|
||||||
|
os.mkdir(logdir)
|
||||||
|
|
||||||
# add the main section
|
dnfbase = dnf.Base()
|
||||||
section = "main"
|
conf = dnfbase.conf
|
||||||
data = {"cachedir": cachedir,
|
conf.logdir = logdir
|
||||||
"keepcache": 0,
|
conf.cachedir = cachedir
|
||||||
"gpgcheck": 0,
|
|
||||||
"plugins": 0,
|
# Turn off logging to the console
|
||||||
"reposdir": "",
|
conf.debuglevel = 0
|
||||||
"tsflags": "nodocs"}
|
conf.errorlevel = 0
|
||||||
|
dnfbase.logging.setup_from_dnf_conf(conf)
|
||||||
|
|
||||||
|
conf.releasever = releasever
|
||||||
|
conf.installroot = installroot
|
||||||
|
conf.prepend_installroot('persistdir')
|
||||||
|
conf.tsflags.append('nodocs')
|
||||||
|
|
||||||
if proxy:
|
if proxy:
|
||||||
data["proxy"] = proxy
|
conf.proxy = proxy
|
||||||
|
|
||||||
if excludepkgs:
|
# add the repositories
|
||||||
data["exclude"] = " ".join(excludepkgs)
|
for i, r in enumerate(repositories):
|
||||||
|
repo_name = "lorax-repo-%d" % i
|
||||||
c.add_section(section)
|
repo = dnf.repo.Repo(repo_name, cachedir)
|
||||||
map(lambda (key, value): c.set(section, key, value), data.items())
|
repo.baseurl = [r]
|
||||||
|
if proxy:
|
||||||
# add the main repository - the first repository from list
|
repo.proxy = proxy
|
||||||
section = "lorax-repo"
|
repo.enable()
|
||||||
data = {"name": "lorax repo",
|
dnfbase.repos.add(repo)
|
||||||
"baseurl": repositories[0],
|
log.info("Added '%s': %s", repo_name, r)
|
||||||
"enabled": 1}
|
log.info("Fetching metadata...")
|
||||||
|
try:
|
||||||
c.add_section(section)
|
repo.load()
|
||||||
map(lambda (key, value): c.set(section, key, value), data.items())
|
except dnf.exceptions.RepoError as e:
|
||||||
|
log.error("Error fetching metadata for %s: %s", repo_name, e)
|
||||||
# add the extra repositories
|
return None
|
||||||
for n, extra in enumerate(repositories[1:], start=1):
|
|
||||||
section = "lorax-extra-repo-{0:d}".format(n)
|
|
||||||
data = {"name": "lorax extra repo {0:d}".format(n),
|
|
||||||
"baseurl": extra,
|
|
||||||
"enabled": 1}
|
|
||||||
|
|
||||||
c.add_section(section)
|
|
||||||
map(lambda (key, value): c.set(section, key, value), data.items())
|
|
||||||
|
|
||||||
# add the mirrorlists
|
# add the mirrorlists
|
||||||
for n, mirror in enumerate(mirrorlists, start=1):
|
for i, r in enumerate(mirrorlists):
|
||||||
section = "lorax-mirrorlist-{0:d}".format(n)
|
repo_name = "lorax-mirrorlist-%d" % i
|
||||||
data = {"name": "lorax mirrorlist {0:d}".format(n),
|
repo = dnf.repo.Repo(repo_name, cachedir)
|
||||||
"mirrorlist": mirror,
|
repo.mirrorlist = r
|
||||||
"enabled": 1 }
|
if proxy:
|
||||||
|
repo.proxy = proxy
|
||||||
|
repo.enable()
|
||||||
|
dnfbase.repos.add(repo)
|
||||||
|
log.info("Added '%s': %s", repo_name, r)
|
||||||
|
log.info("Fetching metadata...")
|
||||||
|
try:
|
||||||
|
repo.load()
|
||||||
|
except dnf.exceptions.RepoError as e:
|
||||||
|
log.error("Error fetching metadata for %s: %s", repo_name, e)
|
||||||
|
return None
|
||||||
|
|
||||||
c.add_section(section)
|
dnfbase.fill_sack(load_system_repo=False)
|
||||||
map(lambda (key, value): c.set(section, key, value), data.items())
|
dnfbase.read_comps()
|
||||||
|
|
||||||
# write the yum configuration file
|
return dnfbase
|
||||||
with open(yumconf, "w") as f:
|
|
||||||
c.write(f)
|
|
||||||
|
|
||||||
# create the yum base object
|
|
||||||
yb = yum.YumBase()
|
|
||||||
|
|
||||||
yb.preconf.fn = yumconf
|
|
||||||
yb.preconf.root = installroot
|
|
||||||
#yb.repos.setCacheDir(cachedir)
|
|
||||||
|
|
||||||
# Turn on as much yum logging as we can
|
|
||||||
yb.preconf.debuglevel = 6
|
|
||||||
yb.preconf.errorlevel = 6
|
|
||||||
yb.logger.setLevel(logging.DEBUG)
|
|
||||||
yb.verbose_logger.setLevel(logging.DEBUG)
|
|
||||||
|
|
||||||
return yb
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user