From 1350cd028f48530fc3836addebc7d6bd01fbd93b Mon Sep 17 00:00:00 2001 From: Will Woods Date: Mon, 9 May 2011 10:47:25 -0400 Subject: [PATCH] treebuilder.py: uses templates to create trees/images TreeBuilder uses templates full of commands (like ramdisk.ltmpl) to create the output tree and boot images. There are 4 arch-specific templates, plus a bonus EFI template which can handle EFI image creation for any arch that implements EFI. --- share/efi.tmpl | 37 +++++ share/ppc.tmpl | 99 ++++++++++++++ share/s390.tmpl | 26 ++++ share/sparc.tmpl | 26 ++++ share/x86.tmpl | 68 ++++++++++ src/pylorax/treebuilder.py | 268 +++++++++++++++++++++++++++++++++++++ 6 files changed, 524 insertions(+) create mode 100644 share/efi.tmpl create mode 100644 share/ppc.tmpl create mode 100644 share/s390.tmpl create mode 100644 share/sparc.tmpl create mode 100644 share/x86.tmpl create mode 100644 src/pylorax/treebuilder.py diff --git a/share/efi.tmpl b/share/efi.tmpl new file mode 100644 index 00000000..9ee64a08 --- /dev/null +++ b/share/efi.tmpl @@ -0,0 +1,37 @@ +<%page args="ANABOOTDIR, KERNELDIR, efiarch"/> +<% EFIBOOTDIR="EFI/BOOT" %> + +mkdir ${EFIBOOTDIR} +install boot/efi/EFI/redhat/grub.efi ${EFIBOOTDIR}/BOOT${efiarch}.bin +install boot/grub/splash.xml.gz ${EFIBOOTDIR} + +## actually make the EFI images +${make_efiboot("images/efidisk.img", include_kernel=True, disk=True)} +${make_efiboot("images/efiboot.img", include_kernel=False)} + +## This is kinda gross, but then... so's EFI. +<%def name="make_efiboot(img, include_kernel, disk=False)"> + <% + kdir = EFIBOOTDIR if include_kernel else KERNELDIR + args = "--label=ANACONDA" + if disk: args += " --disk" + %> + %if include_kernel: + copy ${KERNELDIR}/vmlinuz ${EFIBOOTDIR} + copy ${KERNELDIR}/initrd ${EFIBOOTDIR} + %endif + install ${ANABOOTDIR}/grub.conf ${EFIBOOTDIR}/BOOT${efiarch}.conf + replace @PRODUCT@ ${product.name} ${EFIBOOTDIR}/BOOT${efiarch}.conf + replace @VERSION@ ${product.version} ${EFIBOOTDIR}/BOOT${efiarch}.conf + replace @KERNELPATH@ /${kdir}/vmlinuz ${EFIBOOTDIR}/BOOT${efiarch}.conf + replace @INITRDPATH@ /${kdir}/initrd ${EFIBOOTDIR}/BOOT${efiarch}.conf + replace @SPLASHPATH@ /EFI/BOOT/splash.xpm.gz ${EFIBOOTDIR}/BOOT${efiarch}.conf + %if efiarch == 'IA32': + copy ${EFIBOOTDIR}/BOOT${efiarch}.conf ${EFIBOOTDIR}/BOOT.conf + %endif + runcmd mkefiboot ${args} ${outroot}/${EFIBOOTDIR} ${outroot}/${img} + %if include_kernel: + remove ${EFIBOOTDIR}/vmlinuz + remove ${EFIBOOTDIR}/initrd + %endif + diff --git a/share/ppc.tmpl b/share/ppc.tmpl new file mode 100644 index 00000000..c6302f30 --- /dev/null +++ b/share/ppc.tmpl @@ -0,0 +1,99 @@ +<% +ANABOOTDIR="usr/share/anaconda/boot" +BOOTDIR="ppc" +MACDIR=BOOTDIR+"/mac" +NETBOOTDIR="images/netboot" + +MKZIMAGE="usr/bin/mkzimage" +ZSTUB="usr/share/ppc64-utils/zImage.stub" +WRAPPER="usr/sbin/wrapper" +WRAPPER_A="usr/"+libdir+"/kernel-wrapper/wrapper.a" +MAPPING=ANABOOTDIR+"/mapping" +MAGIC=ANABOOTDIR+"/magic" +bitsizes = set() +prepboot = "" +macboot = "" +%> + +mkdir ${BOOTDIR} ${BOOTDIR}/chrp etc +install ${ANABOOTDIR}/bootinfo.txt ${BOOTDIR} +install boot/efika.forth ${BOOTDIR} +install usr/lib/yaboot/yaboot ${BOOTDIR}/chrp + +## Mac boot stuff +mkdir ${MACDIR} +install usr/lib/yaboot/yaboot ${MACDIR} +install ofboot.b ${MACDIR} +<% + macboot = "-hfs-volid {0}".format(product.version) + macboot += "-hfs-bless {0}/isopath/{1}".format(outroot,MACDIR) +%> + +%for kernel in kernels: + <% + bits = 64 if kernel.arch == "ppc64" else 32 + KERNELDIR=BOOTDIR+"/ppc%s" % bits + NETIMG=NETBOOTDIR+"/ppc%s.img" % bits + bitsizes.add(bits) + %> + mkdir ${KERNELDIR} + install ${ANABOOTDIR}/yaboot.conf.in ${KERNELDIR}/yaboot.conf + installkernel images-${kernel.arch} ${kernel.path} ${KERNELDIR}/vmlinuz + ## Note: this used to be ramdisk.img.gz + installinitrd images-${kernel.arch} ${kernel.initrd.path} ${KERNELDIR}/initrd.img + + replace %PRODUCT% ${product.name} ${KERNELDIR}/yaboot.conf + replace %VERSION% ${product.version} ${KERNELDIR}/yaboot.conf + replace %BITS% ${bits} ${KERNELDIR}/yaboot.conf + + ## Weirdo wrapper junk that makes the netboot combined ppc{32,64}.img + %if exists(MKZIMAGE) and exists(ZSTUB): + copy usr/${libdir}/kernel-wrapper/zImage.lds ${KERNELDIR} + runcmd chdir=${KERNELDIR} ${inroot}/${MKZIMAGE} vmlinuz no no initrd.img \ + ${inroot}/${ZSTUB} ${outroot}/${NETIMG} + remove ${KERNELDIR}/zImage.lds + treeinfo images-${kernel.arch} zimage ${NETIMG} + %elif exists(WRAPPER) and exists(WRAPPER_A): + runcmd chdir=${KERNELDIR} ${inroot}/${WRAPPER} -o ${outroot}/${NETIMG} \ + -i initrd.img -D ${inroot}/${os.path.dirname(WRAPPER_A)} vmlinuz + treeinfo images-${kernel.arch} zimage ${NETIMG} + %endif + %if exists(NETIMG) and bits == 32: + <% prepboot="-prep-boot " + NETIMG %> + %endif +%endfor + +%if not exists(NETBOOTDIR+"/*.img"): + remove ${NETBOOTDIR} +%endif + +runcmd usr/lib/yaboot/addnote ${outroot}/${BOOTDIR}/chrp/yaboot + +%if len(bitsizes) == 2: + ## magic ppc biarch tree! we need magic ppc biarch config. + install ${ANABOOTDIR}/yaboot.conf.3264 etc/yaboot.conf + replace %PRODUCT% ${product.name} etc/yaboot.conf + replace %VERSION% ${product.version} etc/yaboot.conf + replace %BITS% 32 etc/yaboot.conf +%else: + copy ${KERNELDIR}/yaboot.conf etc/yaboot.conf +%endif + +## XXX why don't we use graft-points here? +## is it because of the scary warnings in mkisofs(1)? +mkdir isopath +copy ${BOOTDIR} isopath +copy etc isopath +runcmd mkisofs -o ${outroot}/images/boot.iso -chrp-boot -U \ + ${prepboot} -part -hfs -T -r -l -J \ + -A "${product.name} ${product.version}" -sysid PPC -V "PBOOT" \ + -volset "${product.version}" -volset-size 1 -volset-seqno 1 \ + ${macboot} -map ${MAPPING} -magic ${MAGIC} \ + -no-desktop -allow-multidot -graft-points ${outroot}/isopath +remove isopath +%if len(bitsizes) == 2: + treeinfo images-ppc boot.iso images/boot.iso + treeinfo images-ppc64 boot.iso images/boot.iso +%else: + treeinfo images-${kernel.arch} boot.iso images/boot.iso +%fi diff --git a/share/s390.tmpl b/share/s390.tmpl new file mode 100644 index 00000000..533a7b2b --- /dev/null +++ b/share/s390.tmpl @@ -0,0 +1,26 @@ +<% +ANABOOTDIR="usr/share/anaconda/boot" +BOOTDIR="images" +KERNELDIR=BOOTDIR +INITRD_ADDRESS="0x02000000" +MKCDBOOT="usr/libexec/anaconda/mk-s390-cdboot" +# The assumption seems to be that there is only one s390 kernel, ever +kernel = kernels[0] +%> + +install ${ANABOOTDIR}/redhat.exec ${BOOTDIR} +install ${ANABOOTDIR}/generic.prm ${BOOTDIR} +install ${ANABOOTDIR}/generic.ins . + +replace @INITRD_LOAD_ADDRESS@ ${INITRD_ADDRESS} generic.ins + +installkernel images-${basearch} ${kernel.path} ${KERNELDIR}/kernel.img +installinitrd images-${basearch} ${kernel.initrd.path} ${KERNELDIR}/initrd.img +runcmd usr/libexec/anaconda/addrsize ${INITRD_ADDRESS} ${KERNELDIR}/initrd.img ${outroot}/${BOOTDIR}/initrd_addrsize +treeinfo images-${basearch} initrd.addrsize ${BOOTDIR}/initrd_addrsize +treeinfo images-${basearch} generic.prm ${BOOTDIR}/generic.prm +treeinfo images-${basearch} generic.ins generic.ins + +runcmd ${MKCDBOOT} -i ${kernel.path} -r ${kernel.initrd.path} \ + -p ${outroot}/${BOOTDIR}/generic.prm \ + -o ${outroot}/${BOOTDIR}/cdboot.img diff --git a/share/sparc.tmpl b/share/sparc.tmpl new file mode 100644 index 00000000..dc4cc6ab --- /dev/null +++ b/share/sparc.tmpl @@ -0,0 +1,26 @@ +<% +ANABOOTDIR="usr/share/anaconda/boot" +BOOTDIR="boot" +KERNELDIR=BOOTDIR +%> + +install boot/*.b ${BOOTDIR} +install ${ANABOOTDIR}/silo.conf ${BOOTDIR} +install ${ANABOOTDIR}/boot.msg ${BOOTDIR}/boot.msg + +replace %VERSION% ${product.version} ${BOOTDIR}/boot.msg +replace %PRODUCT% ${product.name} ${BOOTDIR}/boot.msg + +%for kernel in kernels: + installkernel images-${basearch} ${kernel.path} ${KERNELDIR}/vmlinuz + installinitrd images-${basearch} ${kernel.initrd.path} ${KERNELDIR}/initrd.img +%endfor + +runcmd mkisofs -R -J -T -G /${BOOTDIR}/isofs.b -B ... \ + -s /${BOOTDIR}/silo.conf -r -V "PBOOT" \ + -A "${product.name} ${product.version}" \ + -x Fedora -x repodata \ + -sparc-label "${product.name} ${product.version} Boot Disc" \ + -o ${outroot}/images/boot.iso \ + -graft-points boot=${BOOTDIR} +treeinfo images-${basearch} boot.iso images/boot.iso diff --git a/share/x86.tmpl b/share/x86.tmpl new file mode 100644 index 00000000..7cce090b --- /dev/null +++ b/share/x86.tmpl @@ -0,0 +1,68 @@ +<% +ANABOOTDIR="usr/share/anaconda/boot" +SYSLINUXDIR="usr/share/syslinux" +PXEBOOTDIR="images/pxeboot" +BOOTDIR="isolinux" +KERNELDIR=PXEBOOTDIR +%> + +mkdir ${BOOTDIR} ${KERNELDIR} +install ${SYSLINUXDIR}/isolinux.bin ${BOOTDIR} +install ${ANABOOTDIR}/syslinux.cfg ${BOOTDIR}/isolinux.cfg +install ${ANABOOTDIR}/syslinux-vesa-splash.jpg ${BOOTDIR}/splash.jpg +install ${ANABOOTDIR}/*.msg ${BOOTDIR} +install ${SYSLINUXDIR}/vesamenu.c32 ${BOOTDIR} +install ${ANABOOTDIR}/grub.conf ${BOOTDIR} + +replace @VERSION@ ${product.version} ${BOOTDIR}/*.msg ${BOOTDIR}/grub.conf +replace @PRODUCT@ ${product.name} ${BOOTDIR}/grub.conf + +replace "default linux" "default vesamenu.c32" ${BOOTDIR}/isolinux.cfg +replace "prompt 1" "#prompt 1" ${BOOTDIR}/isolinux.cfg + +%if exists("boot/memtest*"): + install boot/memtest* ${BOOTDIR} + append ${BOOTDIR}/isolinux.cfg "label memtest86" + append ${BOOTDIR}/isolinux.cfg " menu label ^Memory test" + append ${BOOTDIR}/isolinux.cfg " kernel memtest" + append ${BOOTDIR}/isolinux.cfg " append -" +%endif + +%for kernel in kernels: + %if kernel.flavor: + installkernel images-xen ${kernel.path} ${KERNELDIR}/vmlinuz-${kernel.flavor} + installinitrd images-xen ${kernel.initrd.path} ${KERNELDIR}/initrd-${kernel.flavor}.img + %else: + installkernel images-${basearch} ${kernel.path} ${KERNELDIR}/vmlinuz + installinitrd images-${basearch} ${kernel.initrd.path} ${KERNELDIR}/initrd.img + %endif +%endfor + +hardlink ${KERNELDIR}/vmlinuz ${BOOTDIR} +hardlink ${KERNELDIR}/initrd ${BOOTDIR} +%if basearch == 'x86_64': + treeinfo images-xen kernel ${KERNELDIR}/vmlinuz + treeinfo images-xen initrd ${KERNELDIR}/initrd.img +%endif + +## WHeeeeeeee, EFI. +## We could remove the basearch restriction someday.. +<% efiargs=""; efigraft="" %> +%if exists("boot/efi/EFI/redhat/grub.efi") and basearch != 'i386': + <% + efiarch = 'X64' if basearch=='x86_64' else 'IA32' + efiargs="-eltorito-alt-boot -e images/efiboot.img -no-emul-boot" + efigraft="BOOT/EFI={0}/BOOT/EFI".format(outroot) + %> + <%include file="efi.tmpl" args="ANABOOTDIR=ANABOOTDIR, KERNELDIR=KERNELDIR, efiarch=efiarch"/> +%endif + +runcmd mkisofs -v -o ${outroot}/images/boot.iso \ + -b ${BOOTDIR}/isolinux.bin -c ${BOOTDIR}/boot.cat \ + -boot-load-size 4 -boot-info-table ${efiargs} \ + -R -J -V '${product.name}' -T -graft-points \ + ${BOOTDIR}=${outroot}/${BOOTDIR} \ + images=${outroot}/images \ + ${efigraft} +runcmd isohybrid ${outroot}/images/boot.iso +treeinfo images-${basearch} boot.iso images/boot.iso diff --git a/src/pylorax/treebuilder.py b/src/pylorax/treebuilder.py new file mode 100644 index 00000000..b7b23a94 --- /dev/null +++ b/src/pylorax/treebuilder.py @@ -0,0 +1,268 @@ +# treebuilder.py - handle arch-specific tree building stuff using templates +# +# Copyright (C) 2011 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Author(s): Will Woods + +import logging +logger = logging.getLogger("pylorax.treebuilder") + +import os +import glob +from subprocess import check_call, PIPE + +from sysutils import joinpaths, cpfile, replace, remove +from ltmpl import LoraxTemplate +from base import DataHolder + +templatemap = {'i386': 'x86.tmpl', + 'x86_64': 'x86.tmpl', + 'ppc': 'ppc.tmpl', + 'ppc64': 'ppc.tmpl', + 'sparc': 'sparc.tmpl', + 'sparc64': 'sparc.tmpl', + 's390': 's390.tmpl', + 's390x': 's390.tmpl', + } + +def findkernels(root="/", kdir="boot"): + # To find flavors, awk '/BuildKernel/ { print $4 }' kernel.spec + flavors = ('debug', 'PAE', 'PAEdebug', 'smp', 'xen') + kre = re.compile(r"vmlinuz-(?P.+?\.(?P[a-z0-9_]+)" + r"(\.(?P{0}))?)$".format("|".join(flavors))) + kernels = [] + for f in os.listdir(joinpaths(root, kdir)): + match = kre.match(f) + if match: + kernel = DataHolder(path=joinpaths(kdir, f)) + kernel.update(match.groupdict()) # sets version, arch, flavor + kernels.append(kernel) + + # look for associated initrd/initramfs + for kernel in kernels: + # NOTE: if both exist, the last one found will win + for imgname in ("initrd", "initramfs"): + i = kernel.path.replace("vmlinuz", imgname, 1) + ".img" + if os.path.exists(joinpaths(root, i)): + kernel.initrd = DataHolder(path=i) + + return kernels + +def _exists(root, p): + if p[0] != '/': p = joinpaths(root, p) + return (len(glob.glob(p)) > 0) + +class BaseBuilder(object): + def __init__(self, product, arch, inroot, outroot): + self.arch = arch + self.product = product + self.inroot = inroot + self.outroot = outroot + self.runner = None + + def getdefaults(self): + return dict(arch=self.arch, product=self.product, + inroot=self.inroot, outroot=self.outroot, + basearch=self.arch.basearch, libdir=self.arch.libdir, + exists=lambda p: _exists(self.inroot, p)) + + def runtemplate(self, templatefile, **variables): + for k,v in self.getdefaults(): + variables.setdefault(k,v) # setdefault won't override existing args + t = LoraxTemplate() + logger.info("parsing %s with the following variables", templatefile) + for key, val in variables.items(): + logger.info(" %s: %s", key, val) + template = t.parse(templatefile, variables) + self.runner = TemplateRunner(self.inroot, self.outroot, template) + logger.info("running template commands") + self.runner.run() + +class TreeBuilder(BaseBuilder): + '''Builds the arch-specific boot images. + inroot should be the installtree root (the newly-built runtime dir)''' + def build(self): + self.runtemplate(templatemap[self.arch.basearch], kernels=self.kernels) + self.implantisomd5() + + @property + def treeinfo_data(self): + if self.runner: + return self.runner.treeinfo_data + + @property + def kernels(self): + return findkernels(root=self.inroot) + + def rebuild_initrds(self, add_args=[], backup=""): + '''Rebuild all the initrds in the tree. If backup is specified, each + initrd will be renamed with backup as a suffix before rebuilding. + If backup is empty, the existing initrd files will be overwritten.''' + dracut = ["/sbin/dracut", "--nomdadmconf", "--nolvmconf"] + add_args + if not backup: + dracut.append("--force") + for kernel in self.kernels: + if backup: + initrd = joinpaths(self.inroot, kernel.initrd.path) + os.rename(initrd, initrd + backup) + check_call(["chroot", self.inroot] + \ + dracut + [kernel.initrd.path, kernel.version]) + + def initrd_append(rootdir): + '''Place the given files into a cpio archive and append that archive + to the initrds.''' + cpio = NamedTemporaryFile(prefix="lorax.") + mkcpio(rootdir, cpio, compression=None) + for kernel in self.kernels: + initrd = open(kernel.initrd.path, "ab") + cpio = open(cpio, "rb") + initrd.write(cpio.read()) + + def implantisomd5(self): + for section, data in self.treeinfo_data: + if 'boot.iso' in data: + iso = joinpaths(self.outputdir, data['boot.iso']) + check_call(["implantisomd5", iso]) + + +# note: "install", "replace", "exists" allow globs +# "install" and "exist" assume their first argument is in inroot +# everything else operates on outroot +# "mkdir", "treeinfo", "runcmd", "remove", "replace" will take multiple args + +# TODO: replace installtree. need glob(), find(glob), installpkg, removepkg, module +# also: run_transaction? + +class TemplateRunner(object): + commands = ('install', 'mkdir', 'replace', 'append', 'treeinfo', + 'installkernel', 'installinitrd', 'hardlink', 'symlink', + 'copy', 'copyif', 'move', 'moveif', 'remove', 'chmod', + 'runcmd', 'log') + + def __init__(self, inroot, outroot, parsed_template, fatalerrors=False): + self.inroot = inroot + self.outroot = outroot + self.template = parsed_template + self.fatalerrors = fatalerrors + + self.treeinfo_data = dict() + self.exists = lambda p: _exists(inroot, p) + + def _out(self, path): + return joinpaths(self.outroot, path) + def _in(self, path): + return joinpaths(self.inroot, path) + + def run(self): + for (num, line) in enumerate(self.template,1): + logger.debug("template line %i: %s", num, line) + (cmd, args) = (line[0], line[1:]) + try: + if cmd not in self.commands: + raise ValueError, "unknown command %s" % cmd + # grab the method named in cmd and pass it the given arguments + f = getattr(self, cmd) + f(*args) + except Exception as e: + logger.error("template command error: %s", str(line)) + if self.fatalerrors: + raise + logger.error(str(e)) + + def install(self, srcglob, dest): + sources = glob.glob(self._in(srcglob)) + if not sources: + raise IOError, "couldn't find %s" % srcglob + for src in sources: + cpfile(src, self._out(dest)) + + def mkdir(self, *dirs): + for d in dirs: + d = self._out(d) + if not os.path.isdir(d): + os.makedirs(d) + + def replace(self, pat, repl, *files): + for f in files: + replace(pat, repl, self._out(f)) + + def append(self, filename, data): + with open(self._out(filename), "a") as fobj: + fobj.write(data+"\n") + + def treeinfo(self, section, key, *valuetoks): + if section not in self.treeinfo: + self.treeinfo_data[section] = dict() + self.treeinfo_data[section][key] = " ".join(valuetoks) + + def installkernel(self, section, src, dest): + self.install(src, dest) + self.treeinfo(section, "kernel", dest) + + def installinitrd(self, section, src, dest): + self.install(src, dest) + self.treeinfo(section, "initrd", dest) + + def hardlink(self, src, dest): + os.link(self._out(src), self._out(dest)) + + def symlink(self, target, dest): + os.symlink(target, self._out(dest)) + + def copy(self, src, dest): + cpfile(self._out(src), self._out(dest)) + + def copyif(self, src, dest): + if self.exists(src): + self.copy(src, dest) + return True + + def move(self, src, dest): + self.copy(src, dest) + self.remove(src) + + def moveif(self, src, dest): + if self.copyif(src, dest): + self.remove(src) + return True + + def remove(self, *targets): + for t in targets: + remove(self._out(t)) + + def chmod(self, target, mode): + os.chmod(self._out(target), int(mode,8)) + + def gconfset(self, path, keytype, value, outfile=None): + if outfile is None: + outfile = self._out("etc/gconf/gconf.xml.defaults") + check_call(["gconftool-2", "--direct", + "--config-source=xml:readwrite:%s" % outfile, + "--set", "--type", keytype, path, value]) + + def log(self, msg): + logger.info(msg) + + def runcmd(self, *cmdlist): + '''Note that we need full paths for everything here''' + chdir = lambda: None + cmd = cmdlist + if cmd[0].startswith("chdir="): + dirname = cmd[0].split('=',1)[1] + chdir = lambda: os.chdir(dirname) + cmd = cmd[1:] + logger.info("runcmd: %s", cmd) + check_call(cmd, preexec_fn=chdir)