lorax/src/pylorax/treebuilder.py

401 lines
17 KiB
Python
Raw Normal View History

# treebuilder.py - handle arch-specific tree building stuff using templates
#
# Copyright (C) 2011-2015 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Author(s): Will Woods <wwoods@redhat.com>
import logging
logger = logging.getLogger("pylorax.treebuilder")
import os, re
2014-05-09 00:21:34 +00:00
from os.path import basename
from shutil import copytree, copy2
from pathlib import Path
import itertools
2014-05-09 00:21:34 +00:00
from pylorax.sysutils import joinpaths, remove
from pylorax.base import DataHolder
from pylorax.ltmpl import LoraxTemplateRunner
import pylorax.imgutils as imgutils
from pylorax.executils import runcmd, runcmd_output, execWithCapture
templatemap = {
'i386': 'x86.tmpl',
'x86_64': 'x86.tmpl',
'ppc': 'ppc.tmpl',
'ppc64': 'ppc.tmpl',
'ppc64le': 'ppc64le.tmpl',
's390': 's390.tmpl',
's390x': 's390.tmpl',
'aarch64': 'aarch64.tmpl',
'arm': 'arm.tmpl',
'armhfp': 'arm.tmpl',
}
def generate_module_info(moddir, outfile=None):
def module_desc(mod):
output = runcmd_output(["modinfo", "-F", "description", mod])
return output.strip()
def read_module_set(name):
return set(l.strip() for l in open(joinpaths(moddir,name)) if ".ko" in l)
modsets = {'scsi':read_module_set("modules.block"),
'eth':read_module_set("modules.networking")}
modinfo = list()
2014-05-09 00:21:34 +00:00
for root, _dirs, files in os.walk(moddir):
for modtype, modset in modsets.items():
for mod in modset.intersection(files): # modules in this dir
2014-05-09 00:21:34 +00:00
(name, _ext) = os.path.splitext(mod) # foo.ko -> (foo, .ko)
desc = module_desc(joinpaths(root,mod)) or "%s driver" % name
modinfo.append(dict(name=name, type=modtype, desc=desc))
out = open(outfile or joinpaths(moddir,"module-info"), "w")
out.write("Version 0\n")
for mod in sorted(modinfo, key=lambda m: m.get('name')):
out.write('{name}\n\t{type}\n\t"{desc:.65}"\n'.format(**mod))
2011-05-14 07:27:25 +00:00
class RuntimeBuilder(object):
'''Builds the anaconda runtime image.'''
def __init__(self, product, arch, dbo, templatedir=None,
installpkgs=None, excludepkgs=None,
add_templates=None,
add_template_vars=None):
root = dbo.conf.installroot
# use a copy of product so we can modify it locally
product = product.copy()
product.name = product.name.lower()
self.vars = DataHolder(arch=arch, product=product, dbo=dbo, root=root,
basearch=arch.basearch, libdir=arch.libdir)
self.dbo = dbo
self._runner = LoraxTemplateRunner(inroot=root, outroot=root,
dbo=dbo, templatedir=templatedir)
self.add_templates = add_templates or []
self.add_template_vars = add_template_vars or {}
self._installpkgs = installpkgs or []
self._excludepkgs = excludepkgs or []
self._runner.defaults = self.vars
self.dbo.reset()
2011-05-12 21:17:50 +00:00
def _install_branding(self):
release = None
q = self.dbo.sack.query()
a = q.available()
for pkg in a.filter(provides='system-release'):
logger.debug("Found release package %s", pkg)
if pkg.name.startswith('generic'):
continue
else:
release = pkg.name
break
if not release:
logger.error('could not get the release')
return
# release
logger.info('got release: %s', release)
self._runner.installpkg(release)
# logos
release, _suffix = release.split('-', 1)
self._runner.installpkg('%s-logos' % release)
2011-05-12 21:17:50 +00:00
def install(self):
'''Install packages and do initial setup with runtime-install.tmpl'''
self._install_branding()
if len(self._installpkgs) > 0:
self._runner.installpkg(*self._installpkgs)
if len(self._excludepkgs) > 0:
self._runner.removepkg(*self._excludepkgs)
self._runner.run("runtime-install.tmpl")
for tmpl in self.add_templates:
self._runner.run(tmpl, **self.add_template_vars)
2011-05-12 21:17:50 +00:00
def writepkglists(self, pkglistdir):
'''debugging data: write out lists of package contents'''
if not os.path.isdir(pkglistdir):
os.makedirs(pkglistdir)
q = self.dbo.sack.query()
for pkgobj in q.installed():
with open(joinpaths(pkglistdir, pkgobj.name), "w") as fobj:
for fname in pkgobj.files:
fobj.write("{0}\n".format(fname))
def postinstall(self):
2011-05-12 21:17:50 +00:00
'''Do some post-install setup work with runtime-postinstall.tmpl'''
# copy configdir into runtime root beforehand
configdir = joinpaths(self._runner.templatedir,"config_files")
configdir_path = "tmp/config_files"
fullpath = joinpaths(self.vars.root, configdir_path)
if os.path.exists(fullpath):
remove(fullpath)
copytree(configdir, fullpath)
self._runner.run("runtime-postinstall.tmpl", configdir=configdir_path)
2011-05-12 21:17:50 +00:00
def cleanup(self):
'''Remove unneeded packages and files with runtime-cleanup.tmpl'''
self._runner.run("runtime-cleanup.tmpl")
2011-05-12 21:17:50 +00:00
def verify(self):
'''Ensure that contents of the installroot can run'''
status = True
ELF_MAGIC = b'\x7fELF'
# Iterate over all files in /usr/bin and /usr/sbin
# For ELF files, gather them into a list and we'll check them all at
# the end. For files with a #!, check them as we go
elf_files = []
usr_bin = Path(self.vars.root + '/usr/bin')
usr_sbin = Path(self.vars.root + '/usr/sbin')
for path in (str(x) for x in itertools.chain(usr_bin.iterdir(), usr_sbin.iterdir()) \
if x.is_file()):
with open(path, "rb") as f:
magic = f.read(4)
if magic == ELF_MAGIC:
# Save the path, minus the chroot prefix
elf_files.append(path[len(self.vars.root):])
elif magic[:2] == b'#!':
# Reopen the file as text and read the first line.
# Open as latin-1 so that stray 8-bit characters don't make
# things blow up. We only really care about ASCII parts.
with open(path, "rt", encoding="latin-1") as f_text:
# Remove the #!, split on space, and take the first part
shabang = f_text.readline()[2:].split()[0]
# Does the path exist?
if not os.path.exists(self.vars.root + shabang):
logger.error('%s, needed by %s, does not exist', shabang, path)
status = False
# Now, run ldd on all the ELF files
# Just run ldd once on everything so it isn't logged a million times.
# At least one thing in the list isn't going to be a dynamic executable,
# so use execWithCapture to ignore the exit code.
filename = ''
for line in execWithCapture('ldd', elf_files, root=self.vars.root,
log_output=False, filter_stderr=True).split('\n'):
if line and not line[0].isspace():
# New filename header, strip the : at the end and save
filename = line[:-1]
elif 'not found' in line:
logger.error('%s, needed by %s, not found', line.split()[0], filename)
status = False
return status
def writepkgsizes(self, pkgsizefile):
'''debugging data: write a big list of pkg sizes'''
fobj = open(pkgsizefile, "w")
getsize = lambda f: os.lstat(f).st_size if os.path.exists(f) else 0
q = self.dbo.sack.query()
for p in sorted(q.installed()):
pkgsize = sum(getsize(joinpaths(self.vars.root,f)) for f in p.files)
fobj.write("{0.name}.{0.arch}: {1}\n".format(p, pkgsize))
def generate_module_data(self):
root = self.vars.root
moddir = joinpaths(root, "lib/modules/")
for kver in os.listdir(moddir):
ksyms = joinpaths(root, "boot/System.map-%s" % kver)
logger.info("doing depmod and module-info for %s", kver)
runcmd(["depmod", "-a", "-F", ksyms, "-b", root, kver])
generate_module_info(moddir+kver, outfile=moddir+"module-info")
2014-05-09 00:21:34 +00:00
def create_runtime(self, outfile="/var/tmp/squashfs.img", compression="xz", compressargs=None, size=2):
# make live rootfs image - must be named "LiveOS/rootfs.img" for dracut
2014-05-09 00:21:34 +00:00
compressargs = compressargs or []
workdir = joinpaths(os.path.dirname(outfile), "runtime-workdir")
os.makedirs(joinpaths(workdir, "LiveOS"))
imgutils.mkrootfsimg(self.vars.root, joinpaths(workdir, "LiveOS/rootfs.img"),
"Anaconda", size=size)
# squash the live rootfs and clean up workdir
imgutils.mksquashfs(workdir, outfile, compression, compressargs)
remove(workdir)
def finished(self):
""" Done using RuntimeBuilder
Close the dnf base object
"""
self.dbo.close()
2011-05-17 19:29:23 +00:00
class TreeBuilder(object):
'''Builds the arch-specific boot images.
inroot should be the installtree root (the newly-built runtime dir)'''
Add ability for external templates to graft content into boot.iso I originally added --add-template to support doing something similar to pungi, which injects content into the system to be used by default. However, this causes the content to be part of the squashfs, which means PXE installations have to download significantly more data that they may not need (if they actually want to pull the tree data from the network, which is not an unusual case). What I actually need is to be able to modify *both* the runtime image and the arch-specific content. For the runtime, I need to change /usr/share/anaconda/interactive-defaults.ks to point to the new content. (Although, potentially we could patch Anaconda itself to auto-detect an ostree repository configured in disk image, similar to what it does for yum repositories) For the arch-specfic image, I want to drop my content into the ISO root. So this patch adds --add-arch-template and --add-arch-template-var in order to do the latter, while preserving the --add-template to affect the runtime image. Further, the templates will automatically graft in a directory named "iso-graft/" from the working directory (if it exists). (I suggest that external templates create a subdirectory named "content" to avoid clashes with any future lorax work) Thus, this will be used by the Atomic Host lorax templates to inject content/repo, but could be used by e.g. pungi to add content/rpms as well. I tried to avoid code deduplication by creating a new template for the product.img bits and this, but that broke because the parent boot.iso code needs access to the `${imggraft}` variable. I think a real fix here would involve turning the product.img, content/, *and* boot.iso into a new template.
2015-03-17 21:26:21 +00:00
def __init__(self, product, arch, inroot, outroot, runtime, isolabel, domacboot=True, doupgrade=True, templatedir=None, add_templates=None, add_template_vars=None, workdir=None):
# NOTE: if you pass an arg named "runtime" to a mako template it'll
# clobber some mako internal variables - hence "runtime_img".
self.vars = DataHolder(arch=arch, product=product, runtime_img=runtime,
runtime_base=basename(runtime),
inroot=inroot, outroot=outroot,
basearch=arch.basearch, libdir=arch.libdir,
Add ability for external templates to graft content into boot.iso I originally added --add-template to support doing something similar to pungi, which injects content into the system to be used by default. However, this causes the content to be part of the squashfs, which means PXE installations have to download significantly more data that they may not need (if they actually want to pull the tree data from the network, which is not an unusual case). What I actually need is to be able to modify *both* the runtime image and the arch-specific content. For the runtime, I need to change /usr/share/anaconda/interactive-defaults.ks to point to the new content. (Although, potentially we could patch Anaconda itself to auto-detect an ostree repository configured in disk image, similar to what it does for yum repositories) For the arch-specfic image, I want to drop my content into the ISO root. So this patch adds --add-arch-template and --add-arch-template-var in order to do the latter, while preserving the --add-template to affect the runtime image. Further, the templates will automatically graft in a directory named "iso-graft/" from the working directory (if it exists). (I suggest that external templates create a subdirectory named "content" to avoid clashes with any future lorax work) Thus, this will be used by the Atomic Host lorax templates to inject content/repo, but could be used by e.g. pungi to add content/rpms as well. I tried to avoid code deduplication by creating a new template for the product.img bits and this, but that broke because the parent boot.iso code needs access to the `${imggraft}` variable. I think a real fix here would involve turning the product.img, content/, *and* boot.iso into a new template.
2015-03-17 21:26:21 +00:00
isolabel=isolabel, udev=udev_escape, domacboot=domacboot, doupgrade=doupgrade,
workdir=workdir, lower=string_lower)
self._runner = LoraxTemplateRunner(inroot, outroot, templatedir=templatedir)
self._runner.defaults = self.vars
Add ability for external templates to graft content into boot.iso I originally added --add-template to support doing something similar to pungi, which injects content into the system to be used by default. However, this causes the content to be part of the squashfs, which means PXE installations have to download significantly more data that they may not need (if they actually want to pull the tree data from the network, which is not an unusual case). What I actually need is to be able to modify *both* the runtime image and the arch-specific content. For the runtime, I need to change /usr/share/anaconda/interactive-defaults.ks to point to the new content. (Although, potentially we could patch Anaconda itself to auto-detect an ostree repository configured in disk image, similar to what it does for yum repositories) For the arch-specfic image, I want to drop my content into the ISO root. So this patch adds --add-arch-template and --add-arch-template-var in order to do the latter, while preserving the --add-template to affect the runtime image. Further, the templates will automatically graft in a directory named "iso-graft/" from the working directory (if it exists). (I suggest that external templates create a subdirectory named "content" to avoid clashes with any future lorax work) Thus, this will be used by the Atomic Host lorax templates to inject content/repo, but could be used by e.g. pungi to add content/rpms as well. I tried to avoid code deduplication by creating a new template for the product.img bits and this, but that broke because the parent boot.iso code needs access to the `${imggraft}` variable. I think a real fix here would involve turning the product.img, content/, *and* boot.iso into a new template.
2015-03-17 21:26:21 +00:00
self.add_templates = add_templates or []
self.add_template_vars = add_template_vars or {}
self.templatedir = templatedir
2014-05-09 00:21:34 +00:00
self.treeinfo_data = None
2011-05-14 07:27:25 +00:00
@property
def kernels(self):
return findkernels(root=self.vars.inroot)
2014-05-09 00:21:34 +00:00
def rebuild_initrds(self, add_args=None, backup="", prefix=""):
'''Rebuild all the initrds in the tree. If backup is specified, each
initrd will be renamed with backup as a suffix before rebuilding.
If backup is empty, the existing initrd files will be overwritten.
If suffix is specified, the existing initrd is untouched and a new
image is built with the filename "${prefix}-${kernel.version}.img"
If the initrd doesn't exist its name will be created based on the
name of the kernel.
'''
2014-05-09 00:21:34 +00:00
add_args = add_args or []
dracut = ["dracut", "--nomdadmconf", "--nolvmconf"] + add_args
if not backup:
dracut.append("--force")
if not self.kernels:
raise Exception("No kernels found, cannot rebuild_initrds")
# Hush some dracut warnings. TODO: bind-mount proc in place?
open(joinpaths(self.vars.inroot,"/proc/modules"),"w")
for kernel in self.kernels:
if prefix:
idir = os.path.dirname(kernel.path)
outfile = joinpaths(idir, prefix+'-'+kernel.version+'.img')
elif hasattr(kernel, "initrd"):
# If there is an existing initrd, use that
outfile = kernel.initrd.path
else:
# Construct an initrd from the kernel name
outfile = kernel.path.replace("vmlinuz-", "initrd-") + ".img"
logger.info("rebuilding %s", outfile)
if backup:
initrd = joinpaths(self.vars.inroot, outfile)
if os.path.exists(initrd):
os.rename(initrd, initrd + backup)
cmd = dracut + [outfile, kernel.version]
runcmd(cmd, root=self.vars.inroot)
# ppc64 cannot boot images > 32MiB, check size and warn
if self.vars.arch.basearch in ("ppc64", "ppc64le") and os.path.exists(outfile):
st = os.stat(outfile)
if st.st_size > 32 * 1024 * 1024:
logging.warning("ppc64 initrd %s is > 32MiB", outfile)
os.unlink(joinpaths(self.vars.inroot,"/proc/modules"))
def build(self):
templatefile = templatemap[self.vars.arch.basearch]
Add ability for external templates to graft content into boot.iso I originally added --add-template to support doing something similar to pungi, which injects content into the system to be used by default. However, this causes the content to be part of the squashfs, which means PXE installations have to download significantly more data that they may not need (if they actually want to pull the tree data from the network, which is not an unusual case). What I actually need is to be able to modify *both* the runtime image and the arch-specific content. For the runtime, I need to change /usr/share/anaconda/interactive-defaults.ks to point to the new content. (Although, potentially we could patch Anaconda itself to auto-detect an ostree repository configured in disk image, similar to what it does for yum repositories) For the arch-specfic image, I want to drop my content into the ISO root. So this patch adds --add-arch-template and --add-arch-template-var in order to do the latter, while preserving the --add-template to affect the runtime image. Further, the templates will automatically graft in a directory named "iso-graft/" from the working directory (if it exists). (I suggest that external templates create a subdirectory named "content" to avoid clashes with any future lorax work) Thus, this will be used by the Atomic Host lorax templates to inject content/repo, but could be used by e.g. pungi to add content/rpms as well. I tried to avoid code deduplication by creating a new template for the product.img bits and this, but that broke because the parent boot.iso code needs access to the `${imggraft}` variable. I think a real fix here would involve turning the product.img, content/, *and* boot.iso into a new template.
2015-03-17 21:26:21 +00:00
for tmpl in self.add_templates:
self._runner.run(tmpl, **self.add_template_vars)
self._runner.run(templatefile, kernels=self.kernels)
2011-06-30 22:13:24 +00:00
self.treeinfo_data = self._runner.results.treeinfo
self.implantisomd5()
def implantisomd5(self):
2014-05-09 00:21:34 +00:00
for _section, data in self.treeinfo_data.items():
if 'boot.iso' in data:
iso = joinpaths(self.vars.outroot, data['boot.iso'])
runcmd(["implantisomd5", iso])
@property
def dracut_hooks_path(self):
""" Return the path to the lorax dracut hooks scripts
Use the configured share dir if it is setup,
otherwise default to /usr/share/lorax/dracut_hooks
"""
if self.templatedir:
return joinpaths(self.templatedir, "dracut_hooks")
else:
return "/usr/share/lorax/dracut_hooks"
def copy_dracut_hooks(self, hooks):
""" Copy the hook scripts in hooks into the installroot's /tmp/
and return a list of commands to pass to dracut when creating the
initramfs
hooks is a list of tuples with the name of the hook script and the
target dracut hook directory
(eg. [("99anaconda-copy-ks.sh", "/lib/dracut/hooks/pre-pivot")])
"""
dracut_commands = []
for hook_script, dracut_path in hooks:
src = joinpaths(self.dracut_hooks_path, hook_script)
if not os.path.exists(src):
2014-05-09 00:21:34 +00:00
logger.error("Missing lorax dracut hook script %s", (src))
continue
dst = joinpaths(self.vars.inroot, "/tmp/", hook_script)
copy2(src, dst)
dracut_commands += ["--include", joinpaths("/tmp/", hook_script),
dracut_path]
return dracut_commands
#### TreeBuilder helper functions
def findkernels(root="/", kdir="boot"):
# To find possible flavors, awk '/BuildKernel/ { print $4 }' kernel.spec
flavors = ('debug', 'PAE', 'PAEdebug', 'smp', 'xen', 'lpae')
kre = re.compile(r"vmlinuz-(?P<version>.+?\.(?P<arch>[a-z0-9_]+)"
r"(.(?P<flavor>{0}))?)$".format("|".join(flavors)))
kernels = []
bootfiles = os.listdir(joinpaths(root, kdir))
for f in bootfiles:
match = kre.match(f)
if match:
kernel = DataHolder(path=joinpaths(kdir, f))
kernel.update(match.groupdict()) # sets version, arch, flavor
kernels.append(kernel)
# look for associated initrd/initramfs/etc.
for kernel in kernels:
for f in bootfiles:
if f.endswith('-'+kernel.version+'.img'):
2014-05-09 00:21:34 +00:00
imgtype, _rest = f.split('-',1)
# special backwards-compat case
if imgtype == 'initramfs':
imgtype = 'initrd'
kernel[imgtype] = DataHolder(path=joinpaths(kdir, f))
2014-05-09 00:21:34 +00:00
logger.debug("kernels=%s", kernels)
return kernels
# udev whitelist: 'a-zA-Z0-9#+.:=@_-' (see is_whitelisted in libudev-util.c)
udev_blacklist=' !"$%&\'()*,/;<>?[\\]^`{|}~' # ASCII printable, minus whitelist
udev_blacklist += ''.join(chr(i) for i in range(32)) # ASCII non-printable
def udev_escape(label):
out = ''
for ch in label:
out += ch if ch not in udev_blacklist else '\\x%02x' % ord(ch)
return out
def string_lower(string):
""" Return a lowercase string.
:param string: String to lowercase
This is used as a filter in the templates.
"""
return string.lower()