lorax/src/pylorax/__init__.py

#
# __init__.py
#
# Copyright (C) 2010  Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# Red Hat Author(s):  Martin Gracik <mgracik@redhat.com>
#                     David Cantrell <dcantrell@redhat.com>
#

# set up logging
import logging
logger = logging.getLogger("pylorax")

sh = logging.StreamHandler()
sh.setLevel(logging.INFO)
logger.addHandler(sh)


import sys
import os
import ConfigParser
import tempfile
import glob
import math
import subprocess

from base import BaseLoraxClass, DataHolder
import output

import yum
import yumhelper
import ltmpl

import imgutils
import constants
from sysutils import *

from treebuilder import TreeBuilder
from installtree import LoraxInstallTree
from buildstamp import BuildStamp
from treeinfo import TreeInfo
from discinfo import DiscInfo

class ArchData(object):
    lib64_arches = ("x86_64", "ppc64", "sparc64", "s390x", "ia64")
    archmap = {"i386": "i386", "i586":"i386", "i686":"i386", "x86_64":"x86_64",
               "ppc":"ppc", "ppc64": "ppc",
               "sparc":"sparc", "sparcv9":"sparc", "sparc64":"sparc",
               "s390":"s390", "s390x":"s390x",
    }
    def __init__(self, buildarch):
        self.buildarch = buildarch
        self.basearch = self.archmap.get(buildarch) or buildarch
        self.libdir = "lib64" if buildarch in self.lib64_arches else "lib"

class Lorax(BaseLoraxClass):

    def __init__(self):
        BaseLoraxClass.__init__(self)
        self._configured = False

    def configure(self, conf_file="/etc/lorax/lorax.conf"):
        self.conf = ConfigParser.SafeConfigParser()

        # set defaults
        self.conf.add_section("lorax")
        self.conf.set("lorax", "debug", "1")
        self.conf.set("lorax", "sharedir", "/usr/share/lorax")

        self.conf.add_section("output")
        self.conf.set("output", "colors", "1")
        self.conf.set("output", "encoding", "utf-8")
        self.conf.set("output", "ignorelist", "/usr/share/lorax/ignorelist")

        self.conf.add_section("templates")
        self.conf.set("templates", "ramdisk", "ramdisk.ltmpl")

        self.conf.add_section("yum")
        self.conf.set("yum", "skipbroken", "0")

        self.conf.add_section("compression")
        self.conf.set("compression", "type", "xz")
        self.conf.set("compression", "speed", "9")

        # read the config file
        if os.path.isfile(conf_file):
            self.conf.read(conf_file)

        # set up the output
        debug = self.conf.getboolean("lorax", "debug")
        output_level = output.DEBUG if debug else output.INFO

        colors = self.conf.getboolean("output", "colors")
        encoding = self.conf.get("output", "encoding")

        self.output.basic_config(output_level=output_level,
                                 colors=colors, encoding=encoding)

        ignorelist = self.conf.get("output", "ignorelist")
        if os.path.isfile(ignorelist):
            with open(ignorelist, "r") as fobj:
                for line in fobj:
                    line = line.strip()
                    if line and not line.startswith("#"):
                        self.output.ignore(line)

        # cron does not have sbin in PATH,
        # so we have to add it ourselves
        os.environ["PATH"] = "{0}:/sbin:/usr/sbin".format(os.environ["PATH"])

        self._configured = True

    def init_file_logging(self, logdir, logname="pylorax.log"):
        fh = logging.FileHandler(filename=joinpaths(logdir, logname), mode="w")
        fh.setLevel(logging.DEBUG)
        logger.addHandler(fh)

    def run(self, ybo, product, version, release, variant="", bugurl="",
            is_beta=False, workdir=None, outputdir=None):

        assert self._configured

        # set up work directory
        self.workdir = workdir or tempfile.mkdtemp(prefix="pylorax.work.")
        if not os.path.isdir(self.workdir):
            os.makedirs(self.workdir)

        # set up log directory
        logdir = joinpaths(self.workdir, "log")
        if not os.path.isdir(logdir):
            os.makedirs(logdir)

        self.init_file_logging(logdir)
        logger.debug("using work directory {0.workdir}".format(self))
        logger.debug("using log directory {0}".format(logdir))

        # set up output directory
        self.outputdir = outputdir or tempfile.mkdtemp(prefix="pylorax.out.")
        if not os.path.isdir(self.outputdir):
            os.makedirs(self.outputdir)
        logger.debug("using output directory {0.outputdir}".format(self))

        # do we have root privileges?
        logger.info("checking for root privileges")
        if not os.geteuid() == 0:
            logger.critical("no root privileges")
            sys.exit(1)

        # do we have all lorax required commands?
        self.lcmds = constants.LoraxRequiredCommands()
        # TODO: actually check for required commands

        # do we have a proper yum base object?
        logger.info("checking yum base object")
        if not isinstance(ybo, yum.YumBase):
            logger.critical("no yum base object")
            sys.exit(1)

        logger.info("setting up yum helper")
        self.yum = yumhelper.LoraxYumHelper(ybo)
        logger.debug("using install root: {0}".format(self.yum.installroot))

        logger.info("setting up build architecture")
        self.arch = ArchData(self.get_buildarch())
        for attr in ('buildarch', 'basearch', 'libdir'):
            logger.debug("self.arch.%s = %s", attr, getattr(self.arch,attr))

        logger.info("setting up install tree")
        self.installtree = LoraxInstallTree(self.yum, self.arch.libdir)

        logger.info("setting up build parameters")
        product = DataHolder(name=product, version=version, release=release,
                             variant=variant, bugurl=bugurl, is_beta=is_beta)
        self.product = product
        logger.debug("product data: %s" % product)

        logger.info("parsing the runtime template")
        tfile = joinpaths(self.conf.get("lorax", "sharedir"),
                          self.conf.get("templates", "ramdisk"))

        # TODO: normalize with arch templates:
        #       tvars = dict(product=product, arch=arch)
        tvars = { "basearch": self.arch.basearch,
                  "buildarch": self.arch.buildarch,
                  "libdir" : self.arch.libdir,
                  "product": self.product.name.lower() }

        template = ltmpl.LoraxTemplate()
        template.parse(tfile, tvars)

        logger.info("creating tree directories")
        for d in template.getdata("mkdir"):
            os.makedirs(joinpaths(self.installtree.root, d))

        # install packages
        logger.info("getting list of required packages")
        for package in template.getdata("install"):
            self.installtree.yum.install(package)

        skipbroken = self.conf.getboolean("yum", "skipbroken")
        self.installtree.yum.process_transaction(skipbroken)

        # write .buildstamp
        buildstamp = BuildStamp(self.product.name, self.product.version,
                                self.product.bugurl, self.product.is_beta, self.arch.buildarch)

        buildstamp.write(joinpaths(self.installtree.root, ".buildstamp"))

        logger.debug("saving pkglists to %s", self.workdir)
        dname = joinpaths(self.workdir, "pkglists")
        os.makedirs(dname)
        for pkgname, pkgobj in self.installtree.yum.installed_packages.items():
            with open(joinpaths(dname, pkgname), "w") as fobj:
                for fname in pkgobj.filelist:
                  fobj.write("{0}\n".format(fname))

        logger.info("removing locales")
        self.installtree.remove_locales()

        logger.info("creating keymaps")
        if self.arch.basearch not in ("s390", "s390x"):
            self.installtree.create_keymaps(basearch=self.arch.basearch)

        logger.info("creating screenfont")
        self.installtree.create_screenfont(basearch=self.arch.basearch)

        logger.info("moving stubs")
        self.installtree.move_stubs()

        logger.info("getting list of required modules")
        # Need a list to pass to cleanup_kernel_modules, not a generator
        modules = list(template.getdata("module"))

        self.installtree.install_kernel_modules(modules)

        logger.info("moving anaconda repos")
        self.installtree.move_repos()

        logger.info("creating depmod.conf")
        self.installtree.create_depmod_conf()

        # misc tree modifications
        if self.arch.basearch in ("s390", "s390x"):
            # TODO: move this to the arch template
            self.installtree.misc_s390_modifications()
        self.installtree.misc_tree_modifications()

        # get config files
        config_dir = joinpaths(self.conf.get("lorax", "sharedir"),
                               "config_files")

        self.installtree.get_config_files(config_dir)
        self.installtree.setup_sshd(config_dir)
        if self.arch.basearch in ("s390", "s390x"):
            self.installtree.generate_ssh_keys()

        # get anaconda portions
        self.installtree.get_anaconda_portions()

        # write .discinfo
        discinfo = DiscInfo(self.product.release, self.arch.basearch)
        discinfo.write(joinpaths(self.outputdir, ".discinfo"))

        # XXX do we need to backup installtree here?

        logger.info("getting list of not required packages")
        removepkgs = template.getdata("remove", mode="lines")
        self.installtree.remove_packages(removepkgs)

        logger.info("cleaning up python files")
        self.installtree.cleanup_python_files()

        logger.info("creating the runtime image")
        # TODO: different img styles / create_runtime implementations
        runtimedir = joinpaths(self.workdir, "runtime")
        # FIXME: compression options (type, speed, etc.)
        create_runtime(self.installtree.root, runtimedir)

        logger.info("preparing to build output tree and boot images")
        treebuilder = TreeBuilder(self.product, self.arch,
                                  self.installtree.root, self.outputdir)

        # TODO: different image styles may do this part differently
        logger.info("rebuilding initramfs images")
        treebuilder.rebuild_initrds(add_args=["--xz", "--add", "btrfs"])

        # TODO: keep small initramfs for split initramfs/runtime media
        logger.info("adding runtime to initrds")
        treebuilder.initrd_append(runtimedir)

        logger.info("populating output tree and building boot images")
        treebuilder.build()

        logger.info("writing .treeinfo")
        treeinfo = TreeInfo(self.product.name, self.product.version,
                            self.product.variant, self.arch.basearch)
        for section, data in treebuilder.treeinfo_data:
            treeinfo.add_section(section, data)
        treeinfo.write(joinpaths(self.outputdir, ".treeinfo"))

    def get_buildarch(self):
        # get architecture of the available anaconda package
        _, available = self.yum.search("anaconda")

        if available:
            anaconda = available.pop(0)
            # src is not a real arch
            if anaconda.arch == "src":
                anaconda = available.pop(0)
            buildarch = anaconda.arch
        else:
            # fallback to the system architecture
            logger.warning("using system architecture")
            buildarch = os.uname()[4]

        return buildarch

def create_runtime(inroot, outdir):
    runtime = "squashfs.img"
    # these exact paths are required by dracut
    livedir = "LiveOS"
    rootfs = "rootfs.img"
    cmdline = "etc/cmdline"
    # make live rootfs image
    fssize = 2 * (1024*1024*1024) # 2GB sparse file compresses down to nothin'
    livedir = joinpaths(outdir, livedir)
    os.makedirs(livedir)
    imgutils.mkext4img(inroot,  joinpaths(livedir, rootfs),
                       label="Anaconda", size=fssize)
    # squash the live image
    imgutils.mksquashfs(livedir, joinpaths(outdir, runtime),
                        compressargs=["-no-progress"])
    remove(livedir)
    # make the cmdline file
    os.makedirs(joinpaths(outdir, os.path.dirname(cmdline)))
    with open(joinpaths(outdir, cmdline), "w") as fobj:
        fobj.write("root=live:/%s\n" % runtime)
    return (runtime, cmdline)