lorax/src/pylorax/__init__.py

#
# __init__.py
#
# Copyright (C) 2010  Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# Red Hat Author(s):  Martin Gracik <mgracik@redhat.com>
#                     David Cantrell <dcantrell@redhat.com>
#                     Will Woods <wwoods@redhat.com>

# set up logging
import logging
logger = logging.getLogger("pylorax")

sh = logging.StreamHandler()
sh.setLevel(logging.INFO)
logger.addHandler(sh)


import sys
import os
import ConfigParser
import tempfile

from base import BaseLoraxClass, DataHolder
import output

import yum
import yumhelper
import ltmpl

import imgutils
import constants
from sysutils import *

from treebuilder import TreeBuilder
from installtree import LoraxInstallTree
from buildstamp import BuildStamp
from treeinfo import TreeInfo
from discinfo import DiscInfo

class ArchData(object):
    lib64_arches = ("x86_64", "ppc64", "sparc64", "s390x", "ia64")
    archmap = {"i386": "i386", "i586":"i386", "i686":"i386", "x86_64":"x86_64",
               "ppc":"ppc", "ppc64": "ppc",
               "sparc":"sparc", "sparcv9":"sparc", "sparc64":"sparc",
               "s390":"s390", "s390x":"s390x",
    }
    def __init__(self, buildarch):
        self.buildarch = buildarch
        self.basearch = self.archmap.get(buildarch) or buildarch
        self.libdir = "lib64" if buildarch in self.lib64_arches else "lib"

class Lorax(BaseLoraxClass):

    def __init__(self):
        BaseLoraxClass.__init__(self)
        self._configured = False

    def configure(self, conf_file="/etc/lorax/lorax.conf"):
        self.conf = ConfigParser.SafeConfigParser()

        # set defaults
        self.conf.add_section("lorax")
        self.conf.set("lorax", "debug", "1")
        self.conf.set("lorax", "sharedir", "/usr/share/lorax")

        self.conf.add_section("output")
        self.conf.set("output", "colors", "1")
        self.conf.set("output", "encoding", "utf-8")
        self.conf.set("output", "ignorelist", "/usr/share/lorax/ignorelist")

        self.conf.add_section("templates")
        self.conf.set("templates", "ramdisk", "ramdisk.ltmpl")

        self.conf.add_section("yum")
        self.conf.set("yum", "skipbroken", "0")

        self.conf.add_section("compression")
        self.conf.set("compression", "type", "xz")
        self.conf.set("compression", "speed", "9")

        # read the config file
        if os.path.isfile(conf_file):
            self.conf.read(conf_file)

        # set up the output
        debug = self.conf.getboolean("lorax", "debug")
        output_level = output.DEBUG if debug else output.INFO

        colors = self.conf.getboolean("output", "colors")
        encoding = self.conf.get("output", "encoding")

        self.output.basic_config(output_level=output_level,
                                 colors=colors, encoding=encoding)

        ignorelist = self.conf.get("output", "ignorelist")
        if os.path.isfile(ignorelist):
            with open(ignorelist, "r") as fobj:
                for line in fobj:
                    line = line.strip()
                    if line and not line.startswith("#"):
                        self.output.ignore(line)

        # cron does not have sbin in PATH,
        # so we have to add it ourselves
        os.environ["PATH"] = "{0}:/sbin:/usr/sbin".format(os.environ["PATH"])

        self._configured = True

    def init_file_logging(self, logdir, logname="pylorax.log"):
        fh = logging.FileHandler(filename=joinpaths(logdir, logname), mode="w")
        fh.setLevel(logging.DEBUG)
        logger.addHandler(fh)

    def run(self, ybo, product, version, release, variant="", bugurl="",
            is_beta=False, workdir=None, outputdir=None):

        assert self._configured

        # set up work directory
        self.workdir = workdir or tempfile.mkdtemp(prefix="pylorax.work.")
        if not os.path.isdir(self.workdir):
            os.makedirs(self.workdir)

        # set up log directory
        logdir = joinpaths(self.workdir, "log")
        if not os.path.isdir(logdir):
            os.makedirs(logdir)

        self.init_file_logging(logdir)
        logger.debug("using work directory {0.workdir}".format(self))
        logger.debug("using log directory {0}".format(logdir))

        # set up output directory
        self.outputdir = outputdir or tempfile.mkdtemp(prefix="pylorax.out.")
        if not os.path.isdir(self.outputdir):
            os.makedirs(self.outputdir)
        logger.debug("using output directory {0.outputdir}".format(self))

        # do we have root privileges?
        logger.info("checking for root privileges")
        if not os.geteuid() == 0:
            logger.critical("no root privileges")
            sys.exit(1)

        # do we have all lorax required commands?
        self.lcmds = constants.LoraxRequiredCommands()
        # TODO: actually check for required commands

        # do we have a proper yum base object?
        logger.info("checking yum base object")
        if not isinstance(ybo, yum.YumBase):
            logger.critical("no yum base object")
            sys.exit(1)

        logger.info("setting up yum helper")
        self.yum = yumhelper.LoraxYumHelper(ybo)
        logger.debug("using install root: {0}".format(self.yum.installroot))

        logger.info("setting up build architecture")
        self.arch = ArchData(self.get_buildarch())
        for attr in ('buildarch', 'basearch', 'libdir'):
            logger.debug("self.arch.%s = %s", attr, getattr(self.arch,attr))

        logger.info("setting up install tree")
        self.installtree = LoraxInstallTree(self.yum, self.arch.libdir)

        logger.info("setting up build parameters")
        product = DataHolder(name=product, version=version, release=release,
                             variant=variant, bugurl=bugurl, is_beta=is_beta)
        self.product = product
        logger.debug("product data: %s" % product)

        logger.info("parsing the runtime template")
        tfile = joinpaths(self.conf.get("lorax", "sharedir"),
                          self.conf.get("templates", "ramdisk"))

        # TODO: normalize with arch templates:
        #       tvars = dict(product=product, arch=arch)
        tvars = { "basearch": self.arch.basearch,
                  "buildarch": self.arch.buildarch,
                  "libdir" : self.arch.libdir,
                  "product": self.product.name.lower() }

        template = ltmpl.LoraxTemplate()
        template.parse(tfile, tvars)

        logger.info("creating tree directories")
        for d in template.getdata("mkdir"):
            os.makedirs(joinpaths(self.installtree.root, d))

        # install packages
        logger.info("getting list of required packages")
        for package in template.getdata("install"):
            self.installtree.yum.install(package)

        skipbroken = self.conf.getboolean("yum", "skipbroken")
        self.installtree.yum.process_transaction(skipbroken)

        # write .buildstamp
        buildstamp = BuildStamp(self.product.name, self.product.version,
                                self.product.bugurl, self.product.is_beta, self.arch.buildarch)

        buildstamp.write(joinpaths(self.installtree.root, ".buildstamp"))

        logger.debug("saving pkglists to %s", self.workdir)
        dname = joinpaths(self.workdir, "pkglists")
        os.makedirs(dname)
        for pkgname, pkgobj in self.installtree.yum.installed_packages.items():
            with open(joinpaths(dname, pkgname), "w") as fobj:
                for fname in pkgobj.filelist:
                  fobj.write("{0}\n".format(fname))

        logger.info("removing locales")
        self.installtree.remove_locales()

        logger.info("creating keymaps")
        if self.arch.basearch not in ("s390", "s390x"):
            self.installtree.create_keymaps(basearch=self.arch.basearch)

        logger.info("creating screenfont")
        self.installtree.create_screenfont(basearch=self.arch.basearch)

        logger.info("moving stubs")
        self.installtree.move_stubs()

        logger.info("getting list of required modules")
        # Need a list to pass to cleanup_kernel_modules, not a generator
        modules = list(template.getdata("module"))

        self.installtree.install_kernel_modules(modules)

        logger.info("moving anaconda repos")
        self.installtree.move_repos()

        logger.info("creating depmod.conf")
        self.installtree.create_depmod_conf()

        # misc tree modifications
        if self.arch.basearch in ("s390", "s390x"):
            # TODO: move this to the arch template
            self.installtree.misc_s390_modifications()
        self.installtree.misc_tree_modifications()

        # get config files
        config_dir = joinpaths(self.conf.get("lorax", "sharedir"),
                               "config_files")

        self.installtree.get_config_files(config_dir)
        self.installtree.setup_sshd(config_dir)
        if self.arch.basearch in ("s390", "s390x"):
            self.installtree.generate_ssh_keys()

        # get anaconda portions
        self.installtree.get_anaconda_portions()

        # write .discinfo
        discinfo = DiscInfo(self.product.release, self.arch.basearch)
        discinfo.write(joinpaths(self.outputdir, ".discinfo"))

        logger.info("backing up installroot")
        installroot = joinpaths(self.workdir, "installroot")
        linktree(self.installtree.root, installroot)

        logger.info("getting list of not required packages")
        removepkgs = template.getdata("remove", mode="lines")
        self.installtree.remove_packages(removepkgs)

        logger.info("cleaning up python files")
        self.installtree.cleanup_python_files()

        logger.info("creating the runtime image")
        # TODO: different img styles / create_runtime implementations
        runtimedir = joinpaths(self.workdir, "runtime")
        # FIXME: compression options (type, speed, etc.)
        create_runtime(self.installtree.root, runtimedir)

        logger.info("preparing to build output tree and boot images")
        treebuilder = TreeBuilder(self.product, self.arch,
                                  installroot, self.outputdir,
                                  templatedir=self.conf.get("lorax", "sharedir"))

        # TODO: different image styles may do this part differently
        logger.info("rebuilding initramfs images")
        treebuilder.rebuild_initrds(add_args=["--xz", "--add", "btrfs"])

        # TODO: keep small initramfs for split initramfs/runtime media
        logger.info("adding runtime to initrds")
        treebuilder.initrd_append(runtimedir)

        logger.info("populating output tree and building boot images")
        treebuilder.build()

        logger.info("writing .treeinfo")
        treeinfo = TreeInfo(self.product.name, self.product.version,
                            self.product.variant, self.arch.basearch)
        for section, data in treebuilder.treeinfo_data.items():
            treeinfo.add_section(section, data)
        treeinfo.write(joinpaths(self.outputdir, ".treeinfo"))

    def get_buildarch(self):
        # get architecture of the available anaconda package
        _, available = self.yum.search("anaconda")

        if available:
            anaconda = available.pop(0)
            # src is not a real arch
            if anaconda.arch == "src":
                anaconda = available.pop(0)
            buildarch = anaconda.arch
        else:
            # fallback to the system architecture
            logger.warning("using system architecture")
            buildarch = os.uname()[4]

        return buildarch

def create_runtime(inroot, outdir):
    runtime = "squashfs.img"
    # these exact paths are required by dracut
    livedir = "LiveOS"
    rootfs = "rootfs.img"
    cmdline = "etc/cmdline"
    # make live rootfs image
    fssize = 2 * (1024*1024*1024) # 2GB sparse file compresses down to nothin'
    livedir = joinpaths(outdir, livedir)
    os.makedirs(livedir)
    imgutils.mkext4img(inroot,  joinpaths(livedir, rootfs),
                       label="Anaconda", size=fssize)
    # squash the live image
    imgutils.mksquashfs(livedir, joinpaths(outdir, runtime),
                        compressargs=["-no-progress"])
    remove(livedir)
    # make the cmdline file
    os.makedirs(joinpaths(outdir, os.path.dirname(cmdline)))
    with open(joinpaths(outdir, cmdline), "w") as fobj:
        fobj.write("root=live:/%s\n" % runtime)
    return (runtime, cmdline)