Compare commits

...

9 Commits

Author SHA1 Message Date
Brian C. Lane
2741e300e4 Skip using srpm repos
DNF has a bug (#1191289) when the SRPM repo isn't the first one used,
and we don't need them for lorax anyway.

(cherry picked from commit 62de8c26f8)
2015-02-16 08:49:26 -08:00
Brian C. Lane
0a1f702091 Drop the dnf Base object deletion code and use reset
It appears that reset+fill_sack will now do the right thing and load the
state of the installed packages. Drop the hack with deleting the object.

Also add a double-check to make sure there really is a list of files
for anaconda-core before we run off and make an image without removing
anything.

(cherry picked from commit 13de711548)
2015-02-16 08:49:26 -08:00
Brian C. Lane
575ecc09ba Get the log directory from the configfile
In the lorax script create it if missing and override it with the
path of --logfile

(cherry picked from commit 9ff5897eee)
2015-02-16 08:49:26 -08:00
Brian C. Lane
9b89f31533 lorax: Add --cachedir, --force and --workdir cmdline options
--cachedir allows the user to specify where the DNF cache is located.
This doesn't actually appear to do much since dnf erases the cache when
it is done. May be useful in the future.

--workdir sets the top level directory for lorax to use for installing
packages, creating installtree and installroot. Normally a temporary
directory under /var/tmp.

Note that the workdir will *not* be removed if there is an error setting
up the DNF object.

--force skips checking if the output directory exists, allowing things
like pungi to use lorax to place the output next to the repo tree it has
already created.

(cherry picked from commit fe457ed71b)
2015-02-16 08:49:26 -08:00
Brian C. Lane
5db62adecb Cleanup help alignment
(cherry picked from commit 1ec35938b3)
2015-02-16 08:49:26 -08:00
Brian C. Lane
bd2f2d9f65 dnf: remove files from installed packages
This is a workaround for a current dnf bug, it doesn't update the state
of the packages after they are installed so we tear down the base dnf
object and create a new one pointing to the installroot.

There is an additional issue with the list of files returned, hawkey and
dnf don't appear to make a distinction between files, dirs and ghosted
dirs like yum did, this can result in too much being removed (eg. all of
/etc/selinux/) so we only remove files not directories.

(cherry picked from commit 345cc89ee0)
2015-02-16 08:49:26 -08:00
Brian C. Lane
870c4ab57f Switch lorax to use dnf instead of yum
pylorax users will need to change to using dnf and pass a dnf.Base()
object as the dbo argument instead of a yum object as the yum or ybo
argument. See the lorax script for an example of how to do this.

The lorax cmdline argument --excludepkgs has been removed since dnf
doesn't appear to have any way to support it and packages should be
controlled using templates anyway.

(cherry picked from commit 431ca6cea4)
2015-02-16 08:49:26 -08:00
Brian C. Lane
a612c1ebd0 Fix Source0 for use with github 2015-02-12 17:06:42 -08:00
Brian C. Lane
281218b258 Automatic commit of package [lorax] release [22.5-1]. 2015-02-12 16:42:03 -08:00
8 changed files with 374 additions and 300 deletions

View File

@ -1,14 +1,18 @@
%define debug_package %{nil} %define debug_package %{nil}
Name: lorax Name: lorax
Version: 22.4 Version: 22.5
Release: 1%{?dist} Release: 1%{?dist}
Summary: Tool for creating the anaconda install images Summary: Tool for creating the anaconda install images
Group: Applications/System Group: Applications/System
License: GPLv2+ License: GPLv2+
URL: https://github.com/rhinstaller/lorax URL: https://github.com/rhinstaller/lorax
Source0: https://github.com/rhinstaller/%{name}/archive/%{name}-${version}-%{release}.tar.gz # To generate Source0 do:
# git clone https://github.com/rhinstaller/lorax
# git checkout -b archive-branch lorax-%%{version}-%%{release}
# tito build --tgz
Source0: %{name}-%{version}.tar.gz
BuildRequires: python2-devel BuildRequires: python2-devel
@ -33,9 +37,9 @@ Requires: squashfs-tools >= 4.2
Requires: util-linux Requires: util-linux
Requires: xz Requires: xz
Requires: pigz Requires: pigz
Requires: yum
Requires: pykickstart Requires: pykickstart
Requires: dracut >= 030 Requires: dracut >= 030
Requires: dnf
%if 0%{?fedora} %if 0%{?fedora}
# Fedora specific deps # Fedora specific deps
@ -76,7 +80,7 @@ including live isos and disk images. It can use libvirtd for the install, or
Anaconda's image install feature. Anaconda's image install feature.
%prep %prep
%setup -q %setup -q -n %{name}-%{version}
%build %build
@ -102,6 +106,11 @@ make DESTDIR=$RPM_BUILD_ROOT mandir=%{_mandir} install
%{_mandir}/man1/*.1* %{_mandir}/man1/*.1*
%changelog %changelog
* Thu Feb 12 2015 Brian C. Lane <bcl@redhat.com> 22.5-1
- os-release moved to /usr/lib (#1191713) (bcl@redhat.com)
- Use /usr/bin/python2 in scripts (bcl@redhat.com)
- Add bridge-utils (#1188812) (bcl@redhat.com)
* Fri Feb 06 2015 Brian C. Lane <bcl@redhat.com> 22.4-1 * Fri Feb 06 2015 Brian C. Lane <bcl@redhat.com> 22.4-1
- livemedia-creator: Add --timeout option to cancel install after X minutes - livemedia-creator: Add --timeout option to cancel install after X minutes
(bcl@redhat.com) (bcl@redhat.com)

View File

@ -1 +1 @@
22.4-1 ./ 22.5-1 ./

View File

@ -36,7 +36,7 @@ import selinux
from pylorax.base import BaseLoraxClass, DataHolder from pylorax.base import BaseLoraxClass, DataHolder
import pylorax.output as output import pylorax.output as output
import yum import dnf
from pylorax.sysutils import joinpaths, remove, linktree from pylorax.sysutils import joinpaths, remove, linktree
from rpmUtils.arch import getBaseArch from rpmUtils.arch import getBaseArch
@ -87,6 +87,7 @@ class Lorax(BaseLoraxClass):
self.conf.add_section("lorax") self.conf.add_section("lorax")
self.conf.set("lorax", "debug", "1") self.conf.set("lorax", "debug", "1")
self.conf.set("lorax", "sharedir", "/usr/share/lorax") self.conf.set("lorax", "sharedir", "/usr/share/lorax")
self.conf.set("lorax", "logdir", "/var/log/lorax")
self.conf.add_section("output") self.conf.add_section("output")
self.conf.set("output", "colors", "1") self.conf.set("output", "colors", "1")
@ -96,9 +97,6 @@ class Lorax(BaseLoraxClass):
self.conf.add_section("templates") self.conf.add_section("templates")
self.conf.set("templates", "ramdisk", "ramdisk.ltmpl") self.conf.set("templates", "ramdisk", "ramdisk.ltmpl")
self.conf.add_section("yum")
self.conf.set("yum", "skipbroken", "0")
self.conf.add_section("compression") self.conf.add_section("compression")
self.conf.set("compression", "type", "xz") self.conf.set("compression", "type", "xz")
self.conf.set("compression", "args", "") self.conf.set("compression", "args", "")
@ -149,7 +147,7 @@ class Lorax(BaseLoraxClass):
fh.setLevel(logging.DEBUG) fh.setLevel(logging.DEBUG)
logger.addHandler(fh) logger.addHandler(fh)
def run(self, ybo, product, version, release, variant="", bugurl="", def run(self, dbo, product, version, release, variant="", bugurl="",
isfinal=False, workdir=None, outputdir=None, buildarch=None, volid=None, isfinal=False, workdir=None, outputdir=None, buildarch=None, volid=None,
domacboot=True, doupgrade=True, remove_temp=False, domacboot=True, doupgrade=True, remove_temp=False,
installpkgs=None, installpkgs=None,
@ -182,7 +180,7 @@ class Lorax(BaseLoraxClass):
os.makedirs(self.workdir) os.makedirs(self.workdir)
# set up log directory # set up log directory
logdir = '/var/log/lorax' logdir = self.conf.get("lorax", "logdir")
if not os.path.isdir(logdir): if not os.path.isdir(logdir):
os.makedirs(logdir) os.makedirs(logdir)
@ -221,16 +219,16 @@ class Lorax(BaseLoraxClass):
logger.critical("selinux must be disabled or in Permissive mode") logger.critical("selinux must be disabled or in Permissive mode")
sys.exit(1) sys.exit(1)
# do we have a proper yum base object? # do we have a proper dnf base object?
logger.info("checking yum base object") logger.info("checking dnf base object")
if not isinstance(ybo, yum.YumBase): if not isinstance(dbo, dnf.Base):
logger.critical("no yum base object") logger.critical("no dnf base object")
sys.exit(1) sys.exit(1)
self.inroot = ybo.conf.installroot self.inroot = dbo.conf.installroot
logger.debug("using install root: {0}".format(self.inroot)) logger.debug("using install root: {0}".format(self.inroot))
if not buildarch: if not buildarch:
buildarch = get_buildarch(ybo) buildarch = get_buildarch(dbo)
logger.info("setting up build architecture") logger.info("setting up build architecture")
self.arch = ArchData(buildarch) self.arch = ArchData(buildarch)
@ -253,15 +251,14 @@ class Lorax(BaseLoraxClass):
sys.exit(1) sys.exit(1)
templatedir = self.conf.get("lorax", "sharedir") templatedir = self.conf.get("lorax", "sharedir")
# NOTE: rb.root = ybo.conf.installroot (== self.inroot) # NOTE: rb.root = dbo.conf.installroot (== self.inroot)
rb = RuntimeBuilder(product=self.product, arch=self.arch, rb = RuntimeBuilder(product=self.product, arch=self.arch,
yum=ybo, templatedir=templatedir, dbo=dbo, templatedir=templatedir,
installpkgs=installpkgs, installpkgs=installpkgs,
add_templates=add_templates, add_templates=add_templates,
add_template_vars=add_template_vars) add_template_vars=add_template_vars)
logger.info("installing runtime packages") logger.info("installing runtime packages")
rb.yum.conf.skip_broken = self.conf.getboolean("yum", "skipbroken")
rb.install() rb.install()
# write .buildstamp # write .buildstamp
@ -306,6 +303,7 @@ class Lorax(BaseLoraxClass):
rb.create_runtime(joinpaths(installroot,runtime), rb.create_runtime(joinpaths(installroot,runtime),
compression=compression, compressargs=compressargs, compression=compression, compressargs=compressargs,
size=size) size=size)
rb.finished()
logger.info("preparing to build output tree and boot images") logger.info("preparing to build output tree and boot images")
treebuilder = TreeBuilder(product=self.product, arch=self.arch, treebuilder = TreeBuilder(product=self.product, arch=self.arch,
@ -358,10 +356,12 @@ class Lorax(BaseLoraxClass):
remove(self.workdir) remove(self.workdir)
def get_buildarch(ybo): def get_buildarch(dbo):
# get architecture of the available anaconda package # get architecture of the available anaconda package
buildarch = None buildarch = None
for anaconda in ybo.doPackageLists(patterns=["anaconda"]).available: q = dbo.sack.query()
a = q.available()
for anaconda in a.filter(name="anaconda"):
if anaconda.arch != "src": if anaconda.arch != "src":
buildarch = anaconda.arch buildarch = anaconda.arch
break break

107
src/pylorax/dnfhelper.py Normal file
View File

@ -0,0 +1,107 @@
#
# dnfhelper.py
#
# Copyright (C) 2010-2014 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Red Hat Author(s): Martin Gracik <mgracik@redhat.com>
# Brian C. Lane <bcl@redhat.com>
#
import logging
logger = logging.getLogger("pylorax.dnfhelper")
import dnf
import collections
import time
import pylorax.output as output
__all__ = ['LoraxDownloadCallback', 'LoraxRpmCallback']
def _paced(fn):
"""Execute `fn` no more often then every 2 seconds."""
def paced_fn(self, *args):
now = time.time()
if now - self.last_time < 2:
return
self.last_time = now
return fn(self, *args)
return paced_fn
class LoraxDownloadCallback(dnf.callback.DownloadProgress):
def __init__(self):
self.downloads = collections.defaultdict(int)
self.last_time = time.time()
self.total_files = 0
self.total_size = 0
self.pkgno = 0
self.total = 0
self.output = output.LoraxOutput()
@_paced
def _update(self):
msg = "Downloading %(pkgno)s / %(total_files)s RPMs, " \
"%(downloaded)s / %(total_size)s (%(percent)d%%) done.\n"
downloaded = sum(self.downloads.values())
vals = {
'downloaded' : downloaded,
'percent' : int(100 * downloaded/self.total_size),
'pkgno' : self.pkgno,
'total_files' : self.total_files,
'total_size' : self.total_size
}
self.output.write(msg % vals)
def end(self, payload, status, err_msg):
nevra = str(payload)
if status is dnf.callback.STATUS_OK:
self.downloads[nevra] = payload.download_size
self.pkgno += 1
self._update()
return
logger.critical("Failed to download '%s': %d - %s", nevra, status, err_msg)
def progress(self, payload, done):
nevra = str(payload)
self.downloads[nevra] = done
self._update()
def start(self, total_files, total_size):
self.total_files = total_files
self.total_size = total_size
class LoraxRpmCallback(dnf.callback.LoggingTransactionDisplay):
def __init__(self, queue):
super(LoraxRpmCallback, self).__init__()
self._queue = queue
self._last_ts = None
self.cnt = 0
def event(self, package, action, te_current, te_total, ts_current, ts_total):
if action == self.PKG_INSTALL and te_current == 0:
# do not report same package twice
if self._last_ts == ts_current:
return
self._last_ts = ts_current
msg = '(%d/%d) %s.%s' % \
(ts_current, ts_total, package.name, package.arch)
self.cnt += 1
self._queue.put(('install', msg))
elif action == self.TRANS_POST:
self._queue.put(('post', None))

View File

@ -28,15 +28,18 @@ from os.path import basename, isdir
from subprocess import CalledProcessError from subprocess import CalledProcessError
from pylorax.sysutils import joinpaths, cpfile, mvfile, replace, remove from pylorax.sysutils import joinpaths, cpfile, mvfile, replace, remove
from pylorax.yumhelper import LoraxDownloadCallback, LoraxTransactionCallback, LoraxRpmCallback from pylorax.dnfhelper import LoraxDownloadCallback, LoraxRpmCallback
from pylorax.base import DataHolder from pylorax.base import DataHolder
from pylorax.executils import runcmd, runcmd_output from pylorax.executils import runcmd, runcmd_output
from pylorax.imgutils import mkcpio from pylorax.imgutils import mkcpio
import pylorax.output as output
from mako.lookup import TemplateLookup from mako.lookup import TemplateLookup
from mako.exceptions import text_error_template from mako.exceptions import text_error_template
import sys, traceback import sys, traceback
import struct import struct
import dnf
import multiprocessing
class LoraxTemplate(object): class LoraxTemplate(object):
def __init__(self, directories=None): def __init__(self, directories=None):
@ -108,7 +111,7 @@ class LoraxTemplateRunner(object):
This class parses and executes Lorax templates. Sample usage: This class parses and executes Lorax templates. Sample usage:
# install a bunch of packages # install a bunch of packages
runner = LoraxTemplateRunner(inroot=rundir, outroot=rundir, yum=yum_obj) runner = LoraxTemplateRunner(inroot=rundir, outroot=rundir, dbo=dnf_obj)
runner.run("install-packages.ltmpl") runner.run("install-packages.ltmpl")
# modify a runtime dir # modify a runtime dir
@ -145,11 +148,11 @@ class LoraxTemplateRunner(object):
* Commands should raise exceptions for errors - don't use sys.exit() * Commands should raise exceptions for errors - don't use sys.exit()
''' '''
def __init__(self, inroot, outroot, yum=None, fatalerrors=True, def __init__(self, inroot, outroot, dbo=None, fatalerrors=True,
templatedir=None, defaults=None): templatedir=None, defaults=None):
self.inroot = inroot self.inroot = inroot
self.outroot = outroot self.outroot = outroot
self.yum = yum self.dbo = dbo
self.fatalerrors = fatalerrors self.fatalerrors = fatalerrors
self.templatedir = templatedir or "/usr/share/lorax" self.templatedir = templatedir or "/usr/share/lorax"
self.templatefile = None self.templatefile = None
@ -166,8 +169,14 @@ class LoraxTemplateRunner(object):
return joinpaths(self.inroot, path) return joinpaths(self.inroot, path)
def _filelist(self, *pkgs): def _filelist(self, *pkgs):
pkglist = self.yum.doPackageLists(pkgnarrow="installed", patterns=pkgs) """ Return the list of files in the packages """
return set([f for pkg in pkglist.installed for f in pkg.filelist+pkg.ghostlist]) pkglist = []
for pkg_glob in pkgs:
pkglist += list(self.dbo.sack.query().installed().filter(name__glob=pkg_glob))
# dnf/hawkey doesn't make any distinction between file, dir or ghost like yum did
# so only return the files.
return set(f for pkg in pkglist for f in pkg.files if not os.path.isdir(self._out(f)))
def _getsize(self, *files): def _getsize(self, *files):
return sum(os.path.getsize(self._out(f)) for f in files if os.path.isfile(self._out(f))) return sum(os.path.getsize(self._out(f)) for f in files if os.path.isfile(self._out(f)))
@ -447,9 +456,9 @@ class LoraxTemplateRunner(object):
cmd = cmd[1:] cmd = cmd[1:]
try: try:
output = runcmd_output(cmd, cwd=cwd) stdout = runcmd_output(cmd, cwd=cwd)
if output: if stdout:
logger.debug('command output:\n%s', output) logger.debug('command output:\n%s', stdout)
logger.debug("command finished successfully") logger.debug("command finished successfully")
except CalledProcessError as e: except CalledProcessError as e:
if e.output: if e.output:
@ -471,7 +480,7 @@ class LoraxTemplateRunner(object):
for p in pkgs: for p in pkgs:
try: try:
self.yum.install(pattern=p) self.dbo.install(p)
except Exception as e: # pylint: disable=broad-except except Exception as e: # pylint: disable=broad-except
# FIXME: save exception and re-raise after the loop finishes # FIXME: save exception and re-raise after the loop finishes
logger.error("installpkg %s failed: %s", p, str(e)) logger.error("installpkg %s failed: %s", p, str(e))
@ -501,19 +510,59 @@ class LoraxTemplateRunner(object):
Actually install all the packages requested by previous 'installpkg' Actually install all the packages requested by previous 'installpkg'
commands. commands.
''' '''
self.yum.buildTransaction()
dl_callback = LoraxDownloadCallback()
self.yum.repos.setProgressBar(dl_callback)
self.yum.processTransaction(callback=LoraxTransactionCallback(dl_callback),
rpmDisplay=LoraxRpmCallback())
# verify if all packages that were supposed to be installed, def do_transaction(base, queue):
# are really installed try:
errs = [t.po for t in self.yum.tsInfo if not self.yum.rpmdb.contains(po=t.po)] display = LoraxRpmCallback(queue)
for po in errs: base.do_transaction(display=display)
logger.error("package '%s' was not installed", po) except BaseException as e:
logger.error("The transaction process has ended abruptly: %s", e)
queue.put(('quit', str(e)))
self.yum.closeRpmDB() try:
logger.info("Checking dependencies")
self.dbo.resolve()
except dnf.exceptions.DepsolveError as e:
logger.error("Dependency check failed: %s", e)
raise
logger.info("%d packages selected", len(self.dbo.transaction))
if len(self.dbo.transaction) == 0:
raise Exception("No packages in transaction")
pkgs_to_download = self.dbo.transaction.install_set
logger.info("Downloading packages")
progress = LoraxDownloadCallback()
try:
self.dbo.download_packages(pkgs_to_download, progress)
except dnf.exceptions.DownloadError as e:
logger.error("Failed to download the following packages: %s", e)
raise
logger.info("Preparing transaction from installation source")
queue = multiprocessing.Queue()
msgout = output.LoraxOutput()
process = multiprocessing.Process(target=do_transaction, args=(self.dbo, queue))
process.start()
(token, msg) = queue.get()
while token not in ('post', 'quit'):
if token == 'install':
logging.info("%s", msg)
msgout.writeline(msg)
(token, msg) = queue.get()
if token == 'quit':
logger.error("Transaction failed.")
raise Exception("Transaction failed")
logger.info("Performing post-installation setup tasks")
process.join()
# Reset the package sack to pick up the installed packages
self.dbo.reset(repos=False)
self.dbo.fill_sack(load_system_repo=True, load_available_repos=False)
# At this point dnf should know about the installed files. Double check that it really does.
if len(self._filelist("anaconda-core")) == 0:
raise Exception("Failed to reset dbo to installed package set")
def removefrom(self, pkg, *globs): def removefrom(self, pkg, *globs):
''' '''

View File

@ -67,27 +67,30 @@ def generate_module_info(moddir, outfile=None):
class RuntimeBuilder(object): class RuntimeBuilder(object):
'''Builds the anaconda runtime image.''' '''Builds the anaconda runtime image.'''
def __init__(self, product, arch, yum, templatedir=None, def __init__(self, product, arch, dbo, templatedir=None,
installpkgs=None, installpkgs=None,
add_templates=None, add_templates=None,
add_template_vars=None): add_template_vars=None):
root = yum.conf.installroot root = dbo.conf.installroot
# use a copy of product so we can modify it locally # use a copy of product so we can modify it locally
product = product.copy() product = product.copy()
product.name = product.name.lower() product.name = product.name.lower()
self.vars = DataHolder(arch=arch, product=product, yum=yum, root=root, self.vars = DataHolder(arch=arch, product=product, dbo=dbo, root=root,
basearch=arch.basearch, libdir=arch.libdir) basearch=arch.basearch, libdir=arch.libdir)
self.yum = yum self.dbo = dbo
self._runner = LoraxTemplateRunner(inroot=root, outroot=root, self._runner = LoraxTemplateRunner(inroot=root, outroot=root,
yum=yum, templatedir=templatedir) dbo=dbo, templatedir=templatedir)
self.add_templates = add_templates or [] self.add_templates = add_templates or []
self.add_template_vars = add_template_vars or {} self.add_template_vars = add_template_vars or {}
self._installpkgs = installpkgs or [] self._installpkgs = installpkgs or []
self._runner.defaults = self.vars self._runner.defaults = self.vars
self.dbo.reset()
def _install_branding(self): def _install_branding(self):
release = None release = None
for pkg in self.yum.whatProvides('/etc/system-release', None, None): q = self.dbo.sack.query()
a = q.available()
for pkg in a.filter(provides='/etc/system-release'):
if pkg.name.startswith('generic'): if pkg.name.startswith('generic'):
continue continue
else: else:
@ -119,9 +122,10 @@ class RuntimeBuilder(object):
'''debugging data: write out lists of package contents''' '''debugging data: write out lists of package contents'''
if not os.path.isdir(pkglistdir): if not os.path.isdir(pkglistdir):
os.makedirs(pkglistdir) os.makedirs(pkglistdir)
for pkgobj in self.yum.doPackageLists(pkgnarrow='installed').installed: q = self.dbo.sack.query()
for pkgobj in q.installed():
with open(joinpaths(pkglistdir, pkgobj.name), "w") as fobj: with open(joinpaths(pkglistdir, pkgobj.name), "w") as fobj:
for fname in pkgobj.filelist + pkgobj.dirlist: for fname in pkgobj.files:
fobj.write("{0}\n".format(fname)) fobj.write("{0}\n".format(fname))
def postinstall(self): def postinstall(self):
@ -143,8 +147,9 @@ class RuntimeBuilder(object):
'''debugging data: write a big list of pkg sizes''' '''debugging data: write a big list of pkg sizes'''
fobj = open(pkgsizefile, "w") fobj = open(pkgsizefile, "w")
getsize = lambda f: os.lstat(f).st_size if os.path.exists(f) else 0 getsize = lambda f: os.lstat(f).st_size if os.path.exists(f) else 0
for p in sorted(self.yum.doPackageLists(pkgnarrow='installed').installed): q = self.dbo.sack.query()
pkgsize = sum(getsize(joinpaths(self.vars.root,f)) for f in p.filelist) for p in sorted(q.installed()):
pkgsize = sum(getsize(joinpaths(self.vars.root,f)) for f in p.files)
fobj.write("{0.name}.{0.arch}: {1}\n".format(p, pkgsize)) fobj.write("{0.name}.{0.arch}: {1}\n".format(p, pkgsize))
def generate_module_data(self): def generate_module_data(self):
@ -169,6 +174,13 @@ class RuntimeBuilder(object):
imgutils.mksquashfs(workdir, outfile, compression, compressargs) imgutils.mksquashfs(workdir, outfile, compression, compressargs)
remove(workdir) remove(workdir)
def finished(self):
""" Done using RuntimeBuilder
Close the dnf base object
"""
self.dbo.close()
class TreeBuilder(object): class TreeBuilder(object):
'''Builds the arch-specific boot images. '''Builds the arch-specific boot images.
inroot should be the installtree root (the newly-built runtime dir)''' inroot should be the installtree root (the newly-built runtime dir)'''

View File

@ -1,127 +0,0 @@
#
# yumhelper.py
#
# Copyright (C) 2010-2014 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Red Hat Author(s): Martin Gracik <mgracik@redhat.com>
#
import logging
logger = logging.getLogger("pylorax.yumhelper")
import sys
import yum, yum.callbacks, yum.rpmtrans
import pylorax.output as output
__all__ = ['LoraxDownloadCallback', 'LoraxTransactionCallback',
'LoraxRpmCallback']
class LoraxDownloadCallback(yum.callbacks.DownloadBaseCallback):
def __init__(self):
yum.callbacks.DownloadBaseCallback.__init__(self)
self.pkgno = 0
self.total = 0
self.output = output.LoraxOutput()
def updateProgress(self, name, frac, fread, ftime):
"""
Update the progress bar
@param name: filename
@param frac: progress fraction (0 -> 1)
@param fread: formated string containing BytesRead
@param ftime: formated string containing remaining or elapsed time
"""
# Only update when it is finished downloading
if frac < 1:
return
self.pkgno += 1
info = "({0:3d}/{1:3d}) "
info = info.format(self.pkgno, self.total)
infolen, pkglen = len(info), len(name)
if (infolen + pkglen) > self.output.width:
name = "{0}...".format(name[:self.output.width-infolen-3])
msg = "{0}<b>{1}</b>\n".format(info, name)
self.output.write(msg)
class LoraxTransactionCallback(object):
def __init__(self, dl_callback):
self.output = output.LoraxOutput()
self.dl_callback = dl_callback
def event(self, state, data=None):
if state == yum.callbacks.PT_DOWNLOAD:
self.output.write("downloading packages\n")
elif state == yum.callbacks.PT_DOWNLOAD_PKGS:
# Initialize the total number of packages being downloaded
self.dl_callback.total = len(data)
elif state == yum.callbacks.PT_GPGCHECK:
self.output.write("checking package signatures\n")
elif state == yum.callbacks.PT_TEST_TRANS:
self.output.write("running test transaction\n")
elif state == yum.callbacks.PT_TRANSACTION:
self.output.write("running transaction\n")
class LoraxRpmCallback(yum.rpmtrans.RPMBaseCallback):
def __init__(self):
yum.rpmtrans.RPMBaseCallback.__init__(self)
self.output = output.LoraxOutput()
def event(self, package, action, te_current, te_total,
ts_current, ts_total):
action_str = self.action[action].encode("utf-8")
info = "({0:3d}/{1:3d}) [{2:3.0f}%] {3} "
info = info.format(ts_current, ts_total,
float(te_current) / float(te_total) * 100,
action_str.lower())
pkg = "{0}".format(package)
infolen, pkglen = len(info), len(pkg)
if (infolen + pkglen) > self.output.width:
pkg = "{0}...".format(pkg[:self.output.width-infolen-3])
msg = "{0}<b>{1}</b>".format(info, pkg)
# When not outputting to a tty we only want to print it once at the end
if sys.stdout.isatty():
self.output.write(msg + "\r")
if te_current == te_total:
self.output.write("\n")
elif te_current == te_total:
self.output.write(msg + "\n")
def filelog(self, package, action):
if self.fileaction.get(action) == "Installed":
logger.debug("{0} installed successfully".format(package))
def errorlog(self, msg):
logger.warning("RPM transaction error: %s", msg)
def scriptout(self, package, msgs):
if msgs:
logger.info("%s scriptlet output:\n%s", package, msgs)

View File

@ -25,20 +25,16 @@ import logging
log = logging.getLogger("lorax") log = logging.getLogger("lorax")
program_log = logging.getLogger("program") program_log = logging.getLogger("program")
pylorax_log = logging.getLogger("pylorax") pylorax_log = logging.getLogger("pylorax")
yum_log = logging.getLogger("yum") dnf_log = logging.getLogger("dnf")
import sys import sys
import os import os
import tempfile import tempfile
from optparse import OptionParser, OptionGroup from optparse import OptionParser, OptionGroup
import ConfigParser
import shutil import shutil
import yum import dnf
# This is a bit of a hack to short circuit yum's internal logging
# handler setup. We already set one up so we don't need it to run.
yum.logginglevels._added_handlers = True
import pylorax import pylorax
def setup_logging(opts): def setup_logging(opts):
@ -67,12 +63,12 @@ def setup_logging(opts):
fh.setLevel(logging.DEBUG) fh.setLevel(logging.DEBUG)
program_log.addHandler(fh) program_log.addHandler(fh)
# yum logging # dnf logging
yum_log.setLevel(logging.DEBUG) dnf_log.setLevel(logging.DEBUG)
logfile = os.path.abspath(os.path.dirname(opts.logfile))+"/yum.log" logfile = os.path.abspath(os.path.dirname(opts.logfile))+"/dnf.log"
fh = logging.FileHandler(filename=logfile, mode="w") fh = logging.FileHandler(filename=logfile, mode="w")
fh.setLevel(logging.DEBUG) fh.setLevel(logging.DEBUG)
yum_log.addHandler(fh) dnf_log.addHandler(fh)
def main(args): def main(args):
@ -116,9 +112,6 @@ def main(args):
help="config file", metavar="STRING") help="config file", metavar="STRING")
optional.add_option("--proxy", default=None, optional.add_option("--proxy", default=None,
help="repo proxy url:port", metavar="STRING") help="repo proxy url:port", metavar="STRING")
optional.add_option("-e", "--excludepkgs", default=[],
action="append", metavar="STRING",
help="package glob to exclude (may be listed multiple times)")
optional.add_option("-i", "--installpkgs", default=[], optional.add_option("-i", "--installpkgs", default=[],
action="append", metavar="STRING", action="append", metavar="STRING",
help="package glob to install before runtime-install.tmpl runs. (may be listed multiple times)") help="package glob to install before runtime-install.tmpl runs. (may be listed multiple times)")
@ -136,6 +129,12 @@ def main(args):
help="Path to logfile") help="Path to logfile")
optional.add_option("--tmp", default="/var/tmp", optional.add_option("--tmp", default="/var/tmp",
help="Top level temporary directory" ) help="Top level temporary directory" )
optional.add_option("--cachedir", default=None,
help="DNF cache directory. Default is a temporary dir.")
optional.add_option("--workdir", default=None,
help="Work directory, overrides --tmp. Default is a temporary dir under /var/tmp")
optional.add_option("--force", default=False, action="store_true",
help="Run even when the destination directory exists")
optional.add_option("--add-template", dest="add_templates", optional.add_option("--add-template", dest="add_templates",
action="append", help="Additional template to execute", action="append", help="Additional template to execute",
default=[]) default=[])
@ -168,29 +167,42 @@ def main(args):
or not opts.source or not outputdir: or not opts.source or not outputdir:
parser.error("missing one or more required arguments") parser.error("missing one or more required arguments")
if os.path.exists(outputdir): if not opts.force and os.path.exists(outputdir):
parser.error("output directory should not exist.") parser.error("output directory %s should not exist." % outputdir)
opts.logfile = os.path.abspath(opts.logfile) opts.logfile = os.path.abspath(opts.logfile)
if not os.path.exists(os.path.dirname(opts.logfile)):
os.makedirs(os.path.dirname(opts.logfile))
if opts.cachedir:
opts.cachedir = os.path.abspath(opts.cachedir)
if opts.workdir:
opts.workdir = os.path.abspath(opts.workdir)
setup_logging(opts) setup_logging(opts)
if not opts.workdir:
tempfile.tempdir = opts.tmp tempfile.tempdir = opts.tmp
# create the temporary directory for lorax # create the temporary directory for lorax
tempdir = tempfile.mkdtemp(prefix="lorax.", dir=tempfile.gettempdir()) tempdir = tempfile.mkdtemp(prefix="lorax.", dir=tempfile.gettempdir())
else:
tempdir = opts.workdir
if not os.path.exists(tempdir):
os.makedirs(tempdir)
# create the yumbase object
installtree = os.path.join(tempdir, "installtree") installtree = os.path.join(tempdir, "installtree")
if not os.path.exists(installtree):
os.mkdir(installtree) os.mkdir(installtree)
yumtempdir = os.path.join(tempdir, "yum") dnftempdir = os.path.join(tempdir, "dnf")
os.mkdir(yumtempdir) if not os.path.exists(dnftempdir):
os.mkdir(dnftempdir)
yb = get_yum_base_object(installtree, opts.source, opts.mirrorlist, dnfbase = get_dnf_base_object(installtree, opts.source, opts.mirrorlist,
yumtempdir, opts.proxy, opts.excludepkgs) dnftempdir, opts.proxy, opts.version, opts.cachedir)
if yb is None: if dnfbase is None:
print("error: unable to create the yumbase object", file=sys.stderr) print("error: unable to create the dnf base object", file=sys.stderr)
if not opts.workdir:
shutil.rmtree(tempdir) shutil.rmtree(tempdir)
sys.exit(1) sys.exit(1)
@ -204,7 +216,8 @@ def main(args):
# run lorax # run lorax
lorax = pylorax.Lorax() lorax = pylorax.Lorax()
lorax.configure(conf_file=opts.config) lorax.configure(conf_file=opts.config)
lorax.run(yb, opts.product, opts.version, opts.release, lorax.conf.set("lorax", "logdir", os.path.dirname(opts.logfile))
lorax.run(dnfbase, opts.product, opts.version, opts.release,
opts.variant, opts.bugurl, opts.isfinal, opts.variant, opts.bugurl, opts.isfinal,
workdir=tempdir, outputdir=outputdir, buildarch=opts.buildarch, workdir=tempdir, outputdir=outputdir, buildarch=opts.buildarch,
volid=opts.volid, domacboot=opts.domacboot, doupgrade=opts.doupgrade, volid=opts.volid, domacboot=opts.domacboot, doupgrade=opts.doupgrade,
@ -214,9 +227,22 @@ def main(args):
remove_temp=True) remove_temp=True)
def get_yum_base_object(installroot, repositories, mirrorlists=None, def get_dnf_base_object(installroot, repositories, mirrorlists=None,
tempdir="/var/tmp", proxy=None, excludepkgs=None): tempdir="/var/tmp", proxy=None, releasever="21",
cachedir=None):
""" Create a dnf Base object and setup the repositories and installroot
:param string installroot: Full path to the installroot
:param list repositories: List of repositories to use for the installation
:param list mirrorlist: List of mirrors to use
:param string tempdir: Path of temporary directory
:param string proxy: http proxy to use when fetching packages
:param string releasever: Release version to pass to dnf
:param string cachedir: Directory to use for caching packages
If tempdir is not set /var/tmp is used.
If cachedir is None a dnf.cache directory is created inside tmpdir
"""
def sanitize_repo(repo): def sanitize_repo(repo):
"""Convert bare paths to file:/// URIs, and silently reject protocols unhandled by yum""" """Convert bare paths to file:/// URIs, and silently reject protocols unhandled by yum"""
if repo.startswith("/"): if repo.startswith("/"):
@ -227,7 +253,6 @@ def get_yum_base_object(installroot, repositories, mirrorlists=None,
return None return None
mirrorlists = mirrorlists or [] mirrorlists = mirrorlists or []
excludepkgs = excludepkgs or []
# sanitize the repositories # sanitize the repositories
repositories = map(sanitize_repo, repositories) repositories = map(sanitize_repo, repositories)
@ -237,78 +262,77 @@ def get_yum_base_object(installroot, repositories, mirrorlists=None,
repositories = filter(bool, repositories) repositories = filter(bool, repositories)
mirrorlists = filter(bool, mirrorlists) mirrorlists = filter(bool, mirrorlists)
cachedir = os.path.join(tempdir, "yum.cache") if not cachedir:
cachedir = os.path.join(tempdir, "dnf.cache")
if not os.path.isdir(cachedir): if not os.path.isdir(cachedir):
os.mkdir(cachedir) os.mkdir(cachedir)
yumconf = os.path.join(tempdir, "yum.conf") logdir = os.path.join(tempdir, "dnf.logs")
c = ConfigParser.ConfigParser() if not os.path.isdir(logdir):
os.mkdir(logdir)
# add the main section dnfbase = dnf.Base()
section = "main" conf = dnfbase.conf
data = {"cachedir": cachedir, conf.logdir = logdir
"keepcache": 0, conf.cachedir = cachedir
"gpgcheck": 0,
"plugins": 0, # Turn off logging to the console
"reposdir": "", conf.debuglevel = 10
"tsflags": "nodocs"} conf.errorlevel = 0
dnfbase.logging.setup_from_dnf_conf(conf)
conf.releasever = releasever
conf.installroot = installroot
conf.prepend_installroot('persistdir')
conf.tsflags.append('nodocs')
if proxy: if proxy:
data["proxy"] = proxy conf.proxy = proxy
if excludepkgs: # add the repositories
data["exclude"] = " ".join(excludepkgs) for i, r in enumerate(repositories):
if "SRPM" in r or "srpm" in r:
c.add_section(section) log.info("Skipping source repo: %s" % r)
map(lambda (key, value): c.set(section, key, value), data.items()) continue
repo_name = "lorax-repo-%d" % i
# add the main repository - the first repository from list repo = dnf.repo.Repo(repo_name, cachedir)
section = "lorax-repo" repo.baseurl = [r]
data = {"name": "lorax repo", if proxy:
"baseurl": repositories[0], repo.proxy = proxy
"enabled": 1} repo.enable()
dnfbase.repos.add(repo)
c.add_section(section) log.info("Added '%s': %s", repo_name, r)
map(lambda (key, value): c.set(section, key, value), data.items()) log.info("Fetching metadata...")
try:
# add the extra repositories repo.load()
for n, extra in enumerate(repositories[1:], start=1): except dnf.exceptions.RepoError as e:
section = "lorax-extra-repo-{0:d}".format(n) log.error("Error fetching metadata for %s: %s", repo_name, e)
data = {"name": "lorax extra repo {0:d}".format(n), return None
"baseurl": extra,
"enabled": 1}
c.add_section(section)
map(lambda (key, value): c.set(section, key, value), data.items())
# add the mirrorlists # add the mirrorlists
for n, mirror in enumerate(mirrorlists, start=1): for i, r in enumerate(mirrorlists):
section = "lorax-mirrorlist-{0:d}".format(n) if "SRPM" in r or "srpm" in r:
data = {"name": "lorax mirrorlist {0:d}".format(n), log.info("Skipping source repo: %s" % r)
"mirrorlist": mirror, continue
"enabled": 1 } repo_name = "lorax-mirrorlist-%d" % i
repo = dnf.repo.Repo(repo_name, cachedir)
repo.mirrorlist = r
if proxy:
repo.proxy = proxy
repo.enable()
dnfbase.repos.add(repo)
log.info("Added '%s': %s", repo_name, r)
log.info("Fetching metadata...")
try:
repo.load()
except dnf.exceptions.RepoError as e:
log.error("Error fetching metadata for %s: %s", repo_name, e)
return None
c.add_section(section) dnfbase.fill_sack(load_system_repo=False)
map(lambda (key, value): c.set(section, key, value), data.items()) dnfbase.read_comps()
# write the yum configuration file return dnfbase
with open(yumconf, "w") as f:
c.write(f)
# create the yum base object
yb = yum.YumBase()
yb.preconf.fn = yumconf
yb.preconf.root = installroot
#yb.repos.setCacheDir(cachedir)
# Turn on as much yum logging as we can
yb.preconf.debuglevel = 6
yb.preconf.errorlevel = 6
yb.logger.setLevel(logging.DEBUG)
yb.verbose_logger.setLevel(logging.DEBUG)
return yb
if __name__ == "__main__": if __name__ == "__main__":