lorax/src/pylorax/__init__.py

376 lines
14 KiB
Python
Raw Normal View History

2009-08-27 07:19:49 +00:00
#
# __init__.py
2010-01-12 11:45:54 +00:00
#
# Copyright (C) 2010-2015 Red Hat, Inc.
2010-01-12 11:45:54 +00:00
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
2010-02-23 13:20:05 +00:00
# Red Hat Author(s): Martin Gracik <mgracik@redhat.com>
# David Cantrell <dcantrell@redhat.com>
# Will Woods <wwoods@redhat.com>
2009-08-27 07:19:49 +00:00
2010-10-12 16:23:29 +00:00
# set up logging
import logging
2011-01-19 14:37:44 +00:00
logger = logging.getLogger("pylorax")
logger.addHandler(logging.NullHandler())
2010-10-12 16:23:29 +00:00
import sys
import os
import configparser
2010-10-12 16:23:29 +00:00
import tempfile
2012-02-17 10:09:59 +00:00
import locale
from subprocess import CalledProcessError
import selinux
2014-05-09 00:21:34 +00:00
from pylorax.base import BaseLoraxClass, DataHolder
import pylorax.output as output
import dnf
2010-10-12 16:23:29 +00:00
2014-05-09 00:21:34 +00:00
from pylorax.sysutils import joinpaths, remove, linktree
2014-05-09 00:21:34 +00:00
from pylorax.treebuilder import RuntimeBuilder, TreeBuilder
from pylorax.buildstamp import BuildStamp
from pylorax.treeinfo import TreeInfo
from pylorax.discinfo import DiscInfo
from pylorax.executils import runcmd, runcmd_output
2010-11-23 10:14:25 +00:00
# List of drivers to remove on ppc64 arch to keep initrd < 32MiB
REMOVE_PPC64_DRIVERS = "floppy scsi_debug nouveau radeon cirrus mgag200"
REMOVE_PPC64_MODULES = "drm plymouth"
class ArchData(DataHolder):
lib64_arches = ("x86_64", "ppc64", "ppc64le", "s390x", "ia64", "aarch64")
2011-07-22 20:59:56 +00:00
bcj_arch = dict(i386="x86", x86_64="x86",
ppc="powerpc", ppc64="powerpc", ppc64le="powerpc",
arm="arm", armhfp="arm")
def __init__(self, buildarch):
2014-05-09 00:21:34 +00:00
super(ArchData, self).__init__()
self.buildarch = buildarch
self.basearch = dnf.arch.basearch(buildarch)
2011-07-05 17:40:48 +00:00
self.libdir = "lib64" if self.basearch in self.lib64_arches else "lib"
2011-07-22 20:59:56 +00:00
self.bcj = self.bcj_arch.get(self.basearch)
2010-08-17 12:14:36 +00:00
2010-02-23 13:20:05 +00:00
class Lorax(BaseLoraxClass):
2010-10-12 16:23:29 +00:00
def __init__(self):
2010-02-23 13:20:05 +00:00
BaseLoraxClass.__init__(self)
2010-10-12 16:23:29 +00:00
self._configured = False
2014-05-09 00:21:34 +00:00
self.product = None
self.workdir = None
self.arch = None
self.conf = None
self.inroot = None
self.debug = False
self.outputdir = None
2010-10-12 16:23:29 +00:00
2012-02-17 10:09:59 +00:00
# set locale to C
locale.setlocale(locale.LC_ALL, 'C')
2010-10-12 16:23:29 +00:00
def configure(self, conf_file="/etc/lorax/lorax.conf"):
self.conf = configparser.SafeConfigParser()
2010-10-12 16:23:29 +00:00
# set defaults
self.conf.add_section("lorax")
self.conf.set("lorax", "debug", "1")
self.conf.set("lorax", "sharedir", "/usr/share/lorax")
self.conf.set("lorax", "logdir", "/var/log/lorax")
2010-10-12 16:23:29 +00:00
self.conf.add_section("output")
self.conf.set("output", "colors", "1")
self.conf.set("output", "encoding", "utf-8")
self.conf.set("output", "ignorelist", "/usr/share/lorax/ignorelist")
2010-10-12 16:23:29 +00:00
self.conf.add_section("templates")
2010-10-27 09:23:47 +00:00
self.conf.set("templates", "ramdisk", "ramdisk.ltmpl")
2010-10-12 16:23:29 +00:00
self.conf.add_section("compression")
self.conf.set("compression", "type", "xz")
self.conf.set("compression", "args", "")
self.conf.set("compression", "bcj", "on")
2010-10-12 16:23:29 +00:00
# read the config file
if os.path.isfile(conf_file):
self.conf.read(conf_file)
# set up the output
2011-08-09 21:59:04 +00:00
self.debug = self.conf.getboolean("lorax", "debug")
output_level = output.DEBUG if self.debug else output.INFO
2010-10-12 16:23:29 +00:00
if sys.stdout.isatty():
colors = self.conf.getboolean("output", "colors")
else:
colors = False
2010-10-12 16:23:29 +00:00
encoding = self.conf.get("output", "encoding")
self.output.basic_config(output_level=output_level,
colors=colors, encoding=encoding)
ignorelist = self.conf.get("output", "ignorelist")
if os.path.isfile(ignorelist):
with open(ignorelist, "r") as fobj:
for line in fobj:
line = line.strip()
if line and not line.startswith("#"):
self.output.ignore(line)
2010-12-02 11:59:08 +00:00
# cron does not have sbin in PATH,
# so we have to add it ourselves
os.environ["PATH"] = "{0}:/sbin:/usr/sbin".format(os.environ["PATH"])
# remove some environmental variables that can cause problems with package scripts
env_remove = ('DISPLAY', 'DBUS_SESSION_BUS_ADDRESS')
list(os.environ.pop(k) for k in env_remove if k in os.environ)
2010-10-12 16:23:29 +00:00
self._configured = True
def init_stream_logging(self):
sh = logging.StreamHandler()
sh.setLevel(logging.INFO)
logger.addHandler(sh)
2011-01-19 14:37:44 +00:00
def init_file_logging(self, logdir, logname="pylorax.log"):
fh = logging.FileHandler(filename=joinpaths(logdir, logname), mode="w")
fh.setLevel(logging.DEBUG)
logger.addHandler(fh)
def run(self, dbo, product, version, release, variant="", bugurl="",
isfinal=False, workdir=None, outputdir=None, buildarch=None, volid=None,
domacboot=True, doupgrade=True, remove_temp=False,
installpkgs=None,
size=2,
add_templates=None,
Add ability for external templates to graft content into boot.iso I originally added --add-template to support doing something similar to pungi, which injects content into the system to be used by default. However, this causes the content to be part of the squashfs, which means PXE installations have to download significantly more data that they may not need (if they actually want to pull the tree data from the network, which is not an unusual case). What I actually need is to be able to modify *both* the runtime image and the arch-specific content. For the runtime, I need to change /usr/share/anaconda/interactive-defaults.ks to point to the new content. (Although, potentially we could patch Anaconda itself to auto-detect an ostree repository configured in disk image, similar to what it does for yum repositories) For the arch-specfic image, I want to drop my content into the ISO root. So this patch adds --add-arch-template and --add-arch-template-var in order to do the latter, while preserving the --add-template to affect the runtime image. Further, the templates will automatically graft in a directory named "iso-graft/" from the working directory (if it exists). (I suggest that external templates create a subdirectory named "content" to avoid clashes with any future lorax work) Thus, this will be used by the Atomic Host lorax templates to inject content/repo, but could be used by e.g. pungi to add content/rpms as well. I tried to avoid code deduplication by creating a new template for the product.img bits and this, but that broke because the parent boot.iso code needs access to the `${imggraft}` variable. I think a real fix here would involve turning the product.img, content/, *and* boot.iso into a new template.
2015-03-17 21:26:21 +00:00
add_template_vars=None,
add_arch_templates=None,
add_arch_template_vars=None):
2010-10-12 16:23:29 +00:00
assert self._configured
installpkgs = installpkgs or []
# get lorax version
try:
import pylorax.version
except ImportError:
vernum = "devel"
else:
vernum = pylorax.version.num
if domacboot:
try:
runcmd(["rpm", "-q", "hfsplus-tools"])
except CalledProcessError:
logger.critical("you need to install hfsplus-tools to create mac images")
sys.exit(1)
2011-01-19 14:37:44 +00:00
# set up work directory
self.workdir = workdir or tempfile.mkdtemp(prefix="pylorax.work.")
if not os.path.isdir(self.workdir):
os.makedirs(self.workdir)
# set up log directory
logdir = self.conf.get("lorax", "logdir")
2011-01-19 14:37:44 +00:00
if not os.path.isdir(logdir):
os.makedirs(logdir)
self.init_stream_logging()
2011-01-19 14:37:44 +00:00
self.init_file_logging(logdir)
logger.debug("version is %s", vernum)
logger.debug("using work directory %s", self.workdir)
logger.debug("using log directory %s", logdir)
2011-01-19 14:37:44 +00:00
# set up output directory
self.outputdir = outputdir or tempfile.mkdtemp(prefix="pylorax.out.")
if not os.path.isdir(self.outputdir):
os.makedirs(self.outputdir)
logger.debug("using output directory %s", self.outputdir)
2011-01-19 14:37:44 +00:00
2010-10-12 16:23:29 +00:00
# do we have root privileges?
logger.info("checking for root privileges")
if not os.geteuid() == 0:
2010-08-17 12:14:36 +00:00
logger.critical("no root privileges")
sys.exit(1)
# is selinux disabled?
# With selinux in enforcing mode the rpcbind package required for
# dracut nfs module, which is in turn required by anaconda module,
# will not get installed, because it's preinstall scriptlet fails,
# resulting in an incomplete initial ramdisk image.
# The reason is that the scriptlet runs tools from the shadow-utils
# package in chroot, particularly groupadd and useradd to add the
# required rpc group and rpc user. This operation fails, because
# the selinux context on files in the chroot, that the shadow-utils
# tools need to access (/etc/group, /etc/passwd, /etc/shadow etc.),
# is wrong and selinux therefore disallows access to these files.
logger.info("checking the selinux mode")
if selinux.is_selinux_enabled() and selinux.security_getenforce():
logger.critical("selinux must be disabled or in Permissive mode")
sys.exit(1)
# do we have a proper dnf base object?
logger.info("checking dnf base object")
if not isinstance(dbo, dnf.Base):
logger.critical("no dnf base object")
2010-08-17 12:14:36 +00:00
sys.exit(1)
self.inroot = dbo.conf.installroot
logger.debug("using install root: %s", self.inroot)
2010-10-12 16:23:29 +00:00
if not buildarch:
buildarch = get_buildarch(dbo)
2010-10-12 16:23:29 +00:00
logger.info("setting up build architecture")
self.arch = ArchData(buildarch)
for attr in ('buildarch', 'basearch', 'libdir'):
logger.debug("self.arch.%s = %s", attr, getattr(self.arch,attr))
2010-08-17 12:14:36 +00:00
2010-10-12 16:23:29 +00:00
logger.info("setting up build parameters")
product = DataHolder(name=product, version=version, release=release,
variant=variant, bugurl=bugurl, isfinal=isfinal)
2010-02-23 13:20:05 +00:00
self.product = product
2014-05-09 00:21:34 +00:00
logger.debug("product data: %s", product)
2010-10-12 16:23:29 +00:00
# NOTE: if you change isolabel, you need to change pungi to match, or
# the pungi images won't boot.
isolabel = volid or "%s-%s-%s" % (product, version, self.arch.basearch)
if len(isolabel) > 32:
logger.fatal("the volume id cannot be longer than 32 characters")
sys.exit(1)
templatedir = self.conf.get("lorax", "sharedir")
# NOTE: rb.root = dbo.conf.installroot (== self.inroot)
rb = RuntimeBuilder(product=self.product, arch=self.arch,
dbo=dbo, templatedir=templatedir,
installpkgs=installpkgs,
add_templates=add_templates,
add_template_vars=add_template_vars)
logger.info("installing runtime packages")
rb.install()
2010-10-12 16:23:29 +00:00
2010-11-08 12:52:11 +00:00
# write .buildstamp
buildstamp = BuildStamp(self.product.name, self.product.version,
self.product.bugurl, self.product.isfinal, self.arch.buildarch)
2010-11-08 12:52:11 +00:00
buildstamp.write(joinpaths(self.inroot, ".buildstamp"))
2010-11-03 13:11:08 +00:00
if self.debug:
rb.writepkglists(joinpaths(logdir, "pkglists"))
rb.writepkgsizes(joinpaths(logdir, "original-pkgsizes.txt"))
2010-11-03 12:40:03 +00:00
logger.info("doing post-install configuration")
rb.postinstall()
2010-10-19 15:35:50 +00:00
2010-11-03 13:11:08 +00:00
# write .discinfo
discinfo = DiscInfo(self.product.release, self.arch.basearch)
discinfo.write(joinpaths(self.outputdir, ".discinfo"))
logger.info("backing up installroot")
installroot = joinpaths(self.workdir, "installroot")
linktree(self.inroot, installroot)
2010-11-08 12:52:11 +00:00
logger.info("generating kernel module metadata")
rb.generate_module_data()
logger.info("cleaning unneeded files")
rb.cleanup()
if self.debug:
rb.writepkgsizes(joinpaths(logdir, "final-pkgsizes.txt"))
2011-04-28 20:41:24 +00:00
logger.info("creating the runtime image")
runtime = "images/install.img"
compression = self.conf.get("compression", "type")
compressargs = self.conf.get("compression", "args").split() # pylint: disable=no-member
if self.conf.getboolean("compression", "bcj"):
if self.arch.bcj:
compressargs += ["-Xbcj", self.arch.bcj]
else:
logger.info("no BCJ filter for arch %s", self.arch.basearch)
rb.create_runtime(joinpaths(installroot,runtime),
compression=compression, compressargs=compressargs,
size=size)
rb.finished()
logger.info("preparing to build output tree and boot images")
treebuilder = TreeBuilder(product=self.product, arch=self.arch,
inroot=installroot, outroot=self.outputdir,
runtime=runtime, isolabel=isolabel,
domacboot=domacboot, doupgrade=doupgrade,
Add ability for external templates to graft content into boot.iso I originally added --add-template to support doing something similar to pungi, which injects content into the system to be used by default. However, this causes the content to be part of the squashfs, which means PXE installations have to download significantly more data that they may not need (if they actually want to pull the tree data from the network, which is not an unusual case). What I actually need is to be able to modify *both* the runtime image and the arch-specific content. For the runtime, I need to change /usr/share/anaconda/interactive-defaults.ks to point to the new content. (Although, potentially we could patch Anaconda itself to auto-detect an ostree repository configured in disk image, similar to what it does for yum repositories) For the arch-specfic image, I want to drop my content into the ISO root. So this patch adds --add-arch-template and --add-arch-template-var in order to do the latter, while preserving the --add-template to affect the runtime image. Further, the templates will automatically graft in a directory named "iso-graft/" from the working directory (if it exists). (I suggest that external templates create a subdirectory named "content" to avoid clashes with any future lorax work) Thus, this will be used by the Atomic Host lorax templates to inject content/repo, but could be used by e.g. pungi to add content/rpms as well. I tried to avoid code deduplication by creating a new template for the product.img bits and this, but that broke because the parent boot.iso code needs access to the `${imggraft}` variable. I think a real fix here would involve turning the product.img, content/, *and* boot.iso into a new template.
2015-03-17 21:26:21 +00:00
templatedir=templatedir,
add_templates=add_arch_templates,
add_template_vars=add_arch_template_vars,
workdir=self.workdir)
logger.info("rebuilding initramfs images")
dracut_args = ["--xz", "--install", "/.buildstamp"]
anaconda_args = dracut_args + ["--add", "anaconda pollcdrom"]
# ppc64 cannot boot an initrd > 32MiB so remove some drivers
if self.arch.basearch in ("ppc64", "ppc64le"):
dracut_args.extend(["--omit-drivers", REMOVE_PPC64_DRIVERS])
# Only omit dracut modules from the initrd so that they're kept for
# upgrade.img
anaconda_args.extend(["--omit", REMOVE_PPC64_MODULES])
treebuilder.rebuild_initrds(add_args=anaconda_args)
2012-12-18 11:04:03 +00:00
if doupgrade:
# Build upgrade.img. It'd be nice if these could coexist in the same
# image, but that would increase the size of the anaconda initramfs,
# which worries some people (esp. PPC tftpboot). So they're separate.
try:
# If possible, use the 'fedup' plymouth theme
themes = runcmd_output(['plymouth-set-default-theme', '--list'],
root=installroot)
if 'fedup' in themes.splitlines():
os.environ['PLYMOUTH_THEME_NAME'] = 'fedup'
except RuntimeError:
pass
upgrade_args = dracut_args + ["--add", "system-upgrade"]
treebuilder.rebuild_initrds(add_args=upgrade_args, prefix="upgrade")
logger.info("populating output tree and building boot images")
2011-04-28 20:41:24 +00:00
treebuilder.build()
# write .treeinfo file and we're done
treeinfo = TreeInfo(self.product.name, self.product.version,
self.product.variant, self.arch.basearch)
for section, data in treebuilder.treeinfo_data.items():
2011-04-28 20:41:24 +00:00
treeinfo.add_section(section, data)
treeinfo.write(joinpaths(self.outputdir, ".treeinfo"))
2010-11-03 13:11:08 +00:00
# cleanup
if remove_temp:
remove(self.workdir)
def get_buildarch(dbo):
# get architecture of the available anaconda package
buildarch = None
q = dbo.sack.query()
a = q.available()
for anaconda in a.filter(name="anaconda"):
if anaconda.arch != "src":
buildarch = anaconda.arch
break
if not buildarch:
logger.critical("no anaconda package in the repository")
sys.exit(1)
return buildarch