Date: Fri May 15 17:30:02 2015 -0700 Lorax v23.9 Documentation diff --git a/.buildinfo b/.buildinfo new file mode 100644 index 00000000..53c96413 --- /dev/null +++ b/.buildinfo @@ -0,0 +1,4 @@ +# Sphinx build info version 1 +# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. +config: d776326856759e24083884fa308a1bc8 +tags: 645f666f9bcd5a90fca523b33c5a78b7 diff --git a/.doctrees/environment.pickle b/.doctrees/environment.pickle new file mode 100644 index 00000000..004c3d11 Binary files /dev/null and b/.doctrees/environment.pickle differ diff --git a/.doctrees/index.doctree b/.doctrees/index.doctree new file mode 100644 index 00000000..f8f59a4d Binary files /dev/null and b/.doctrees/index.doctree differ diff --git a/.doctrees/intro.doctree b/.doctrees/intro.doctree new file mode 100644 index 00000000..8f6dd3a4 Binary files /dev/null and b/.doctrees/intro.doctree differ diff --git a/.doctrees/livemedia-creator.doctree b/.doctrees/livemedia-creator.doctree new file mode 100644 index 00000000..5ff4e3db Binary files /dev/null and b/.doctrees/livemedia-creator.doctree differ diff --git a/.doctrees/lorax.doctree b/.doctrees/lorax.doctree new file mode 100644 index 00000000..74d4ee94 Binary files /dev/null and b/.doctrees/lorax.doctree differ diff --git a/.doctrees/modules.doctree b/.doctrees/modules.doctree new file mode 100644 index 00000000..0ae4354d Binary files /dev/null and b/.doctrees/modules.doctree differ diff --git a/.doctrees/product-images.doctree b/.doctrees/product-images.doctree new file mode 100644 index 00000000..d3dd62e4 Binary files /dev/null and b/.doctrees/product-images.doctree differ diff --git a/.doctrees/pylorax.doctree b/.doctrees/pylorax.doctree new file mode 100644 index 00000000..b02ed717 Binary files /dev/null and b/.doctrees/pylorax.doctree differ diff --git a/.doctrees/source/index.doctree b/.doctrees/source/index.doctree new file mode 100644 index 00000000..a1aebf96 Binary files /dev/null and b/.doctrees/source/index.doctree differ diff --git a/_modules/index.html b/_modules/index.html new file mode 100644 index 00000000..894ad67e --- /dev/null +++ b/_modules/index.html @@ -0,0 +1,195 @@ + + + + + + + + + + Overview: module code — Lorax 23.8 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+ + + + +

+ + + + + + +

+ +

All modules for which code is available

pylorax

+ +

+ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/_modules/pylorax.html b/_modules/pylorax.html new file mode 100644 index 00000000..d96055bb --- /dev/null +++ b/_modules/pylorax.html @@ -0,0 +1,561 @@ + + + + + + + + + + pylorax — Lorax 23.8 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+ + + + +

+ + + + + + +

+ +

Source code for pylorax

+#
+# __init__.py
+#
+# Copyright (C) 2010-2015  Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Red Hat Author(s):  Martin Gracik <mgracik@redhat.com>
+#                     David Cantrell <dcantrell@redhat.com>
+#                     Will Woods <wwoods@redhat.com>
+
+# set up logging
+import logging
+logger = logging.getLogger("pylorax")
+logger.addHandler(logging.NullHandler())
+
+import sys
+import os
+import configparser
+import tempfile
+import locale
+from subprocess import CalledProcessError
+import selinux
+
+from pylorax.base import BaseLoraxClass, DataHolder
+import pylorax.output as output
+
+import dnf
+
+from pylorax.sysutils import joinpaths, remove, linktree
+
+from pylorax.treebuilder import RuntimeBuilder, TreeBuilder
+from pylorax.buildstamp import BuildStamp
+from pylorax.treeinfo import TreeInfo
+from pylorax.discinfo import DiscInfo
+from pylorax.executils import runcmd, runcmd_output
+
+# List of drivers to remove on ppc64 arch to keep initrd < 32MiB
+REMOVE_PPC64_DRIVERS = "floppy scsi_debug nouveau radeon cirrus mgag200"
+REMOVE_PPC64_MODULES = "drm plymouth"
+
+[docs]class ArchData(DataHolder):
+    lib64_arches = ("x86_64", "ppc64", "ppc64le", "s390x", "ia64", "aarch64")
+    bcj_arch = dict(i386="x86", x86_64="x86",
+                    ppc="powerpc", ppc64="powerpc", ppc64le="powerpc",
+                    arm="arm", armhfp="arm")
+
+    def __init__(self, buildarch):
+        super(ArchData, self).__init__()
+        self.buildarch = buildarch
+        self.basearch = dnf.arch.basearch(buildarch)
+        self.libdir = "lib64" if self.basearch in self.lib64_arches else "lib"
+        self.bcj = self.bcj_arch.get(self.basearch)
+
+[docs]class Lorax(BaseLoraxClass):
+
+    def __init__(self):
+        BaseLoraxClass.__init__(self)
+        self._configured = False
+        self.product = None
+        self.workdir = None
+        self.arch = None
+        self.conf = None
+        self.inroot = None
+        self.debug = False
+        self.outputdir = None
+
+        # set locale to C
+        locale.setlocale(locale.LC_ALL, 'C')
+
+[docs]    def configure(self, conf_file="/etc/lorax/lorax.conf"):
+        self.conf = configparser.SafeConfigParser()
+
+        # set defaults
+        self.conf.add_section("lorax")
+        self.conf.set("lorax", "debug", "1")
+        self.conf.set("lorax", "sharedir", "/usr/share/lorax")
+        self.conf.set("lorax", "logdir", "/var/log/lorax")
+
+        self.conf.add_section("output")
+        self.conf.set("output", "colors", "1")
+        self.conf.set("output", "encoding", "utf-8")
+        self.conf.set("output", "ignorelist", "/usr/share/lorax/ignorelist")
+
+        self.conf.add_section("templates")
+        self.conf.set("templates", "ramdisk", "ramdisk.ltmpl")
+
+        self.conf.add_section("compression")
+        self.conf.set("compression", "type", "xz")
+        self.conf.set("compression", "args", "")
+        self.conf.set("compression", "bcj", "on")
+
+        # read the config file
+        if os.path.isfile(conf_file):
+            self.conf.read(conf_file)
+
+        # set up the output
+        self.debug = self.conf.getboolean("lorax", "debug")
+        output_level = output.DEBUG if self.debug else output.INFO
+
+        if sys.stdout.isatty():
+            colors = self.conf.getboolean("output", "colors")
+        else:
+            colors = False
+        encoding = self.conf.get("output", "encoding")
+
+        self.output.basic_config(output_level=output_level,
+                                 colors=colors, encoding=encoding)
+
+        ignorelist = self.conf.get("output", "ignorelist")
+        if os.path.isfile(ignorelist):
+            with open(ignorelist, "r") as fobj:
+                for line in fobj:
+                    line = line.strip()
+                    if line and not line.startswith("#"):
+                        self.output.ignore(line)
+
+        # cron does not have sbin in PATH,
+        # so we have to add it ourselves
+        os.environ["PATH"] = "{0}:/sbin:/usr/sbin".format(os.environ["PATH"])
+
+        # remove some environmental variables that can cause problems with package scripts
+        env_remove = ('DISPLAY', 'DBUS_SESSION_BUS_ADDRESS')
+        list(os.environ.pop(k) for k in env_remove if k in os.environ)
+
+        self._configured = True
+
+[docs]    def init_stream_logging(self):
+        sh = logging.StreamHandler()
+        sh.setLevel(logging.INFO)
+        logger.addHandler(sh)
+
+[docs]    def init_file_logging(self, logdir, logname="pylorax.log"):
+        fh = logging.FileHandler(filename=joinpaths(logdir, logname), mode="w")
+        fh.setLevel(logging.DEBUG)
+        logger.addHandler(fh)
+
+[docs]    def run(self, dbo, product, version, release, variant="", bugurl="",
+            isfinal=False, workdir=None, outputdir=None, buildarch=None, volid=None,
+            domacboot=True, doupgrade=True, remove_temp=False,
+            installpkgs=None,
+            size=2,
+            add_templates=None,
+            add_template_vars=None,
+            add_arch_templates=None,
+            add_arch_template_vars=None):
+
+        assert self._configured
+
+        installpkgs = installpkgs or []
+
+        # get lorax version
+        try:
+            import pylorax.version
+        except ImportError:
+            vernum = "devel"
+        else:
+            vernum = pylorax.version.num
+
+        if domacboot:
+            try:
+                runcmd(["rpm", "-q", "hfsplus-tools"])
+            except CalledProcessError:
+                logger.critical("you need to install hfsplus-tools to create mac images")
+                sys.exit(1)
+
+        # set up work directory
+        self.workdir = workdir or tempfile.mkdtemp(prefix="pylorax.work.")
+        if not os.path.isdir(self.workdir):
+            os.makedirs(self.workdir)
+
+        # set up log directory
+        logdir = self.conf.get("lorax", "logdir")
+        if not os.path.isdir(logdir):
+            os.makedirs(logdir)
+
+        self.init_stream_logging()
+        self.init_file_logging(logdir)
+
+        logger.debug("version is %s", vernum)
+        logger.debug("using work directory %s", self.workdir)
+        logger.debug("using log directory %s", logdir)
+
+        # set up output directory
+        self.outputdir = outputdir or tempfile.mkdtemp(prefix="pylorax.out.")
+        if not os.path.isdir(self.outputdir):
+            os.makedirs(self.outputdir)
+        logger.debug("using output directory %s", self.outputdir)
+
+        # do we have root privileges?
+        logger.info("checking for root privileges")
+        if not os.geteuid() == 0:
+            logger.critical("no root privileges")
+            sys.exit(1)
+
+        # is selinux disabled?
+        # With selinux in enforcing mode the rpcbind package required for
+        # dracut nfs module, which is in turn required by anaconda module,
+        # will not get installed, because it's preinstall scriptlet fails,
+        # resulting in an incomplete initial ramdisk image.
+        # The reason is that the scriptlet runs tools from the shadow-utils
+        # package in chroot, particularly groupadd and useradd to add the
+        # required rpc group and rpc user. This operation fails, because
+        # the selinux context on files in the chroot, that the shadow-utils
+        # tools need to access (/etc/group, /etc/passwd, /etc/shadow etc.),
+        # is wrong and selinux therefore disallows access to these files.
+        logger.info("checking the selinux mode")
+        if selinux.is_selinux_enabled() and selinux.security_getenforce():
+            logger.critical("selinux must be disabled or in Permissive mode")
+            sys.exit(1)
+
+        # do we have a proper dnf base object?
+        logger.info("checking dnf base object")
+        if not isinstance(dbo, dnf.Base):
+            logger.critical("no dnf base object")
+            sys.exit(1)
+        self.inroot = dbo.conf.installroot
+        logger.debug("using install root: %s", self.inroot)
+
+        if not buildarch:
+            buildarch = get_buildarch(dbo)
+
+        logger.info("setting up build architecture")
+        self.arch = ArchData(buildarch)
+        for attr in ('buildarch', 'basearch', 'libdir'):
+            logger.debug("self.arch.%s = %s", attr, getattr(self.arch,attr))
+
+        logger.info("setting up build parameters")
+        product = DataHolder(name=product, version=version, release=release,
+                             variant=variant, bugurl=bugurl, isfinal=isfinal)
+        self.product = product
+        logger.debug("product data: %s", product)
+
+        # NOTE: if you change isolabel, you need to change pungi to match, or
+        # the pungi images won't boot.
+        isolabel = volid or "%s-%s-%s" % (product, version, self.arch.basearch)
+
+        if len(isolabel) > 32:
+            logger.fatal("the volume id cannot be longer than 32 characters")
+            sys.exit(1)
+
+        templatedir = self.conf.get("lorax", "sharedir")
+        # NOTE: rb.root = dbo.conf.installroot (== self.inroot)
+        rb = RuntimeBuilder(product=self.product, arch=self.arch,
+                            dbo=dbo, templatedir=templatedir,
+                            installpkgs=installpkgs,
+                            add_templates=add_templates,
+                            add_template_vars=add_template_vars)
+
+        logger.info("installing runtime packages")
+        rb.install()
+
+        # write .buildstamp
+        buildstamp = BuildStamp(self.product.name, self.product.version,
+                                self.product.bugurl, self.product.isfinal, self.arch.buildarch)
+
+        buildstamp.write(joinpaths(self.inroot, ".buildstamp"))
+
+        if self.debug:
+            rb.writepkglists(joinpaths(logdir, "pkglists"))
+            rb.writepkgsizes(joinpaths(logdir, "original-pkgsizes.txt"))
+
+        logger.info("doing post-install configuration")
+        rb.postinstall()
+
+        # write .discinfo
+        discinfo = DiscInfo(self.product.release, self.arch.basearch)
+        discinfo.write(joinpaths(self.outputdir, ".discinfo"))
+
+        logger.info("backing up installroot")
+        installroot = joinpaths(self.workdir, "installroot")
+        linktree(self.inroot, installroot)
+
+        logger.info("generating kernel module metadata")
+        rb.generate_module_data()
+
+        logger.info("cleaning unneeded files")
+        rb.cleanup()
+
+        if self.debug:
+            rb.writepkgsizes(joinpaths(logdir, "final-pkgsizes.txt"))
+
+        logger.info("creating the runtime image")
+        runtime = "images/install.img"
+        compression = self.conf.get("compression", "type")
+        compressargs = self.conf.get("compression", "args").split()     # pylint: disable=no-member
+        if self.conf.getboolean("compression", "bcj"):
+            if self.arch.bcj:
+                compressargs += ["-Xbcj", self.arch.bcj]
+            else:
+                logger.info("no BCJ filter for arch %s", self.arch.basearch)
+        rb.create_runtime(joinpaths(installroot,runtime),
+                          compression=compression, compressargs=compressargs,
+                          size=size)
+        rb.finished()
+
+        logger.info("preparing to build output tree and boot images")
+        treebuilder = TreeBuilder(product=self.product, arch=self.arch,
+                                  inroot=installroot, outroot=self.outputdir,
+                                  runtime=runtime, isolabel=isolabel,
+                                  domacboot=domacboot, doupgrade=doupgrade,
+                                  templatedir=templatedir,
+                                  add_templates=add_arch_templates,
+                                  add_template_vars=add_arch_template_vars,
+                                  workdir=self.workdir)
+
+        logger.info("rebuilding initramfs images")
+        dracut_args = ["--xz", "--install", "/.buildstamp"]
+        anaconda_args = dracut_args + ["--add", "anaconda pollcdrom"]
+
+        # ppc64 cannot boot an initrd > 32MiB so remove some drivers
+        if self.arch.basearch in ("ppc64", "ppc64le"):
+            dracut_args.extend(["--omit-drivers", REMOVE_PPC64_DRIVERS])
+
+            # Only omit dracut modules from the initrd so that they're kept for
+            # upgrade.img
+            anaconda_args.extend(["--omit", REMOVE_PPC64_MODULES])
+
+        treebuilder.rebuild_initrds(add_args=anaconda_args)
+
+        if doupgrade:
+            # Build upgrade.img. It'd be nice if these could coexist in the same
+            # image, but that would increase the size of the anaconda initramfs,
+            # which worries some people (esp. PPC tftpboot). So they're separate.
+            try:
+                # If possible, use the 'fedup' plymouth theme
+                themes = runcmd_output(['plymouth-set-default-theme', '--list'],
+                                       root=installroot)
+                if 'fedup' in themes.splitlines():
+                    os.environ['PLYMOUTH_THEME_NAME'] = 'fedup'
+            except RuntimeError:
+                pass
+            upgrade_args = dracut_args + ["--add", "system-upgrade"]
+            treebuilder.rebuild_initrds(add_args=upgrade_args, prefix="upgrade")
+
+        logger.info("populating output tree and building boot images")
+        treebuilder.build()
+
+        # write .treeinfo file and we're done
+        treeinfo = TreeInfo(self.product.name, self.product.version,
+                            self.product.variant, self.arch.basearch)
+        for section, data in treebuilder.treeinfo_data.items():
+            treeinfo.add_section(section, data)
+        treeinfo.write(joinpaths(self.outputdir, ".treeinfo"))
+
+        # cleanup
+        if remove_temp:
+            remove(self.workdir)
+
+
+[docs]def get_buildarch(dbo):
+    # get architecture of the available anaconda package
+    buildarch = None
+    q = dbo.sack.query()
+    a = q.available()
+    for anaconda in a.filter(name="anaconda"):
+        if anaconda.arch != "src":
+            buildarch = anaconda.arch
+            break
+    if not buildarch:
+        logger.critical("no anaconda package in the repository")
+        sys.exit(1)
+
+    return buildarch
+

+ +

+ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/_modules/pylorax/base.html b/_modules/pylorax/base.html new file mode 100644 index 00000000..f815c545 --- /dev/null +++ b/_modules/pylorax/base.html @@ -0,0 +1,252 @@ + + + + + + + + + + pylorax.base — Lorax 23.8 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+ + + + +

+ + + + + + +

+ +

Source code for pylorax.base

+#
+# base.py
+#
+# Copyright (C) 2009-2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Red Hat Author(s):  Martin Gracik <mgracik@redhat.com>
+#
+
+from abc import ABCMeta, abstractmethod
+import sys
+
+import pylorax.output as output
+
+
+[docs]class BaseLoraxClass(object, metaclass=ABCMeta):
+    @abstractmethod
+    def __init__(self):
+        self.output = output.LoraxOutput()
+
+[docs]    def pcritical(self, msg, fobj=sys.stdout):
+        self.output.critical(msg, fobj)
+
+[docs]    def perror(self, msg, fobj=sys.stdout):
+        self.output.error(msg, fobj)
+
+[docs]    def pwarning(self, msg, fobj=sys.stdout):
+        self.output.warning(msg, fobj)
+
+[docs]    def pinfo(self, msg, fobj=sys.stdout):
+        self.output.info(msg, fobj)
+
+[docs]    def pdebug(self, msg, fobj=sys.stdout):
+        self.output.debug(msg, fobj)
+
+
+[docs]class DataHolder(dict):
+
+    def __init__(self, **kwargs):
+        dict.__init__(self)
+
+        for attr, value in kwargs.items():
+            self[attr] = value
+
+    def __getattr__(self, attr):
+        return self[attr]
+
+    def __setattr__(self, attr, value):
+        self[attr] = value
+
+[docs]    def copy(self):
+        return DataHolder(**dict.copy(self))
+

+ +

+ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/_modules/pylorax/buildstamp.html b/_modules/pylorax/buildstamp.html new file mode 100644 index 00000000..537734e3 --- /dev/null +++ b/_modules/pylorax/buildstamp.html @@ -0,0 +1,246 @@ + + + + + + + + + + pylorax.buildstamp — Lorax 23.8 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+ + + + +

+ + + + + + +

+ +

Source code for pylorax.buildstamp

+#
+# buildstamp.py
+#
+# Copyright (C) 2010-2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Red Hat Author(s):  Martin Gracik <mgracik@redhat.com>
+#
+
+import logging
+logger = logging.getLogger("pylorax.buildstamp")
+
+import datetime
+
+
+[docs]class BuildStamp(object):
+
+    def __init__(self, product, version, bugurl, isfinal, buildarch):
+        self.product = product
+        self.version = version
+        self.bugurl = bugurl
+        self.isfinal = isfinal
+
+        now = datetime.datetime.now()
+        now = now.strftime("%Y%m%d%H%M")
+        self.uuid = "{0}.{1}".format(now, buildarch)
+
+[docs]    def write(self, outfile):
+        # get lorax version
+        try:
+            import pylorax.version
+        except ImportError:
+            vernum = "devel"
+        else:
+            vernum = pylorax.version.num
+
+        logger.info("writing .buildstamp file")
+        with open(outfile, "w") as fobj:
+            fobj.write("[Main]\n")
+            fobj.write("Product={0.product}\n".format(self))
+            fobj.write("Version={0.version}\n".format(self))
+            fobj.write("BugURL={0.bugurl}\n".format(self))
+            fobj.write("IsFinal={0.isfinal}\n".format(self))
+            fobj.write("UUID={0.uuid}\n".format(self))
+            fobj.write("[Compose]\n")
+            fobj.write("Lorax={0}\n".format(vernum))
+

+ +

+ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/_modules/pylorax/decorators.html b/_modules/pylorax/decorators.html new file mode 100644 index 00000000..34f44457 --- /dev/null +++ b/_modules/pylorax/decorators.html @@ -0,0 +1,218 @@ + + + + + + + + + + pylorax.decorators — Lorax 23.8 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+ + + + +

+ + + + + + +

+ +

Source code for pylorax.decorators

+#
+# decorators.py
+#
+# Copyright (C) 2009-2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Red Hat Author(s):  Martin Gracik <mgracik@redhat.com>
+#
+
+[docs]def singleton(cls):
+    instances = {}
+
+    def get_instance():
+        if cls not in instances:
+            instances[cls] = cls()
+        return instances[cls]
+
+    return get_instance
+

+ +

+ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/_modules/pylorax/discinfo.html b/_modules/pylorax/discinfo.html new file mode 100644 index 00000000..651b6e50 --- /dev/null +++ b/_modules/pylorax/discinfo.html @@ -0,0 +1,227 @@ + + + + + + + + + + pylorax.discinfo — Lorax 23.8 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+ + + + +

+ + + + + + +

+ +

Source code for pylorax.discinfo

+#
+# discinfo.py
+#
+# Copyright (C) 2010-2015  Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Red Hat Author(s):  Martin Gracik <mgracik@redhat.com>
+#
+
+import logging
+logger = logging.getLogger("pylorax.discinfo")
+
+import time
+
+
+[docs]class DiscInfo(object):
+
+    def __init__(self, release, basearch):
+        self.release = release
+        self.basearch = basearch
+
+[docs]    def write(self, outfile):
+        logger.info("writing .discinfo file")
+        with open(outfile, "w") as fobj:
+            fobj.write("{0:f}\n".format(time.time()))
+            fobj.write("{0.release}\n".format(self))
+            fobj.write("{0.basearch}\n".format(self))
+

+ +

+ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/_modules/pylorax/dnfhelper.html b/_modules/pylorax/dnfhelper.html new file mode 100644 index 00000000..4862ac89 --- /dev/null +++ b/_modules/pylorax/dnfhelper.html @@ -0,0 +1,295 @@ + + + + + + + + + + pylorax.dnfhelper — Lorax 23.8 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+ + + + +

+ + + + + + +

+ +

Source code for pylorax.dnfhelper

+#
+# dnfhelper.py
+#
+# Copyright (C) 2010-2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Red Hat Author(s):  Martin Gracik <mgracik@redhat.com>
+#                     Brian C. Lane <bcl@redhat.com>
+#
+
+import logging
+logger = logging.getLogger("pylorax.dnfhelper")
+import dnf
+import collections
+import time
+import pylorax.output as output
+
+__all__ = ['LoraxDownloadCallback', 'LoraxRpmCallback']
+
+def _paced(fn):
+    """Execute `fn` no more often then every 2 seconds."""
+    def paced_fn(self, *args):
+        now = time.time()
+        if now - self.last_time < 2:
+            return
+        self.last_time = now
+        return fn(self, *args)
+    return paced_fn
+
+
+[docs]class LoraxDownloadCallback(dnf.callback.DownloadProgress):
+    def __init__(self):
+        self.downloads = collections.defaultdict(int)
+        self.last_time = time.time()
+        self.total_files = 0
+        self.total_size = 0
+
+        self.pkgno = 0
+        self.total = 0
+
+        self.output = output.LoraxOutput()
+
+    @_paced
+    def _update(self):
+        msg = "Downloading %(pkgno)s / %(total_files)s RPMs, " \
+              "%(downloaded)s / %(total_size)s (%(percent)d%%) done.\n"
+        downloaded = sum(self.downloads.values())
+        vals = {
+            'downloaded'  : downloaded,
+            'percent'     : int(100 * downloaded/self.total_size),
+            'pkgno'       : self.pkgno,
+            'total_files' : self.total_files,
+            'total_size'  : self.total_size
+        }
+        self.output.write(msg % vals)
+
+[docs]    def end(self, payload, status, err_msg):
+        nevra = str(payload)
+        if status is dnf.callback.STATUS_OK:
+            self.downloads[nevra] = payload.download_size
+            self.pkgno += 1
+            self._update()
+            return
+        logger.critical("Failed to download '%s': %d - %s", nevra, status, err_msg)
+
+[docs]    def progress(self, payload, done):
+        nevra = str(payload)
+        self.downloads[nevra] = done
+        self._update()
+
+[docs]    def start(self, total_files, total_size):
+        self.total_files = total_files
+        self.total_size = total_size
+
+
+[docs]class LoraxRpmCallback(dnf.callback.LoggingTransactionDisplay):
+    def __init__(self, queue):
+        super(LoraxRpmCallback, self).__init__()
+        self._queue = queue
+        self._last_ts = None
+        self.cnt = 0
+
+[docs]    def event(self, package, action, te_current, te_total, ts_current, ts_total):
+        if action == self.PKG_INSTALL and te_current == 0:
+            # do not report same package twice
+            if self._last_ts == ts_current:
+                return
+            self._last_ts = ts_current
+
+            msg = '(%d/%d) %s.%s' % \
+                (ts_current, ts_total, package.name, package.arch)
+            self.cnt += 1
+            self._queue.put(('install', msg))
+        elif action == self.TRANS_POST:
+            self._queue.put(('post', None))
+

+ +

+ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/_modules/pylorax/executils.html b/_modules/pylorax/executils.html new file mode 100644 index 00000000..0581a703 --- /dev/null +++ b/_modules/pylorax/executils.html @@ -0,0 +1,427 @@ + + + + + + + + + + pylorax.executils — Lorax 23.8 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+ + + + +

+ + + + + + +

+ +

Source code for pylorax.executils

+#
+# executil.py - subprocess execution utility functions
+#
+# Copyright (C) 1999-2015
+# Red Hat, Inc.  All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import subprocess
+import signal
+from time import sleep
+
+import logging
+log = logging.getLogger("pylorax")
+program_log = logging.getLogger("program")
+
+from threading import Lock
+program_log_lock = Lock()
+
+_child_env = {}
+
+[docs]def setenv(name, value):
+    """ Set an environment variable to be used by child processes.
+
+        This method does not modify os.environ for the running process, which
+        is not thread-safe. If setenv has already been called for a particular
+        variable name, the old value is overwritten.
+
+        :param str name: The name of the environment variable
+        :param str value: The value of the environment variable
+    """
+
+    _child_env[name] = value
+
+[docs]def augmentEnv():
+    env = os.environ.copy()
+    env.update(_child_env)
+    return env
+
+[docs]class ExecProduct(object):
+    def __init__(self, rc, stdout, stderr):
+        self.rc = rc
+        self.stdout = stdout
+        self.stderr = stderr
+
+[docs]def startProgram(argv, root='/', stdin=None, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+        env_prune=None, env_add=None, reset_handlers=True, reset_lang=True, **kwargs):
+    """ Start an external program and return the Popen object.
+
+        The root and reset_handlers arguments are handled by passing a
+        preexec_fn argument to subprocess.Popen, but an additional preexec_fn
+        can still be specified and will be run. The user preexec_fn will be run
+        last.
+
+        :param argv: The command to run and argument
+        :param root: The directory to chroot to before running command.
+        :param stdin: The file object to read stdin from.
+        :param stdout: The file object to write stdout to.
+        :param stderr: The file object to write stderr to.
+        :param env_prune: environment variables to remove before execution
+        :param env_add: environment variables to add before execution
+        :param reset_handlers: whether to reset to SIG_DFL any signal handlers set to SIG_IGN
+        :param reset_lang: whether to set the locale of the child process to C
+        :param kwargs: Additional parameters to pass to subprocess.Popen
+        :return: A Popen object for the running command.
+    """
+    if env_prune is None:
+        env_prune = []
+
+    # Check for and save a preexec_fn argument
+    preexec_fn = kwargs.pop("preexec_fn", None)
+
+    def preexec():
+        # If a target root was specificed, chroot into it
+        if root and root != '/':
+            os.chroot(root)
+            os.chdir("/")
+
+        # Signal handlers set to SIG_IGN persist across exec. Reset
+        # these to SIG_DFL if requested. In particular this will include the
+        # SIGPIPE handler set by python.
+        if reset_handlers:
+            for signum in range(1, signal.NSIG):
+                if signal.getsignal(signum) == signal.SIG_IGN:
+                    signal.signal(signum, signal.SIG_DFL)
+
+        # If the user specified an additional preexec_fn argument, run it
+        if preexec_fn is not None:
+            preexec_fn()
+
+    with program_log_lock:
+        program_log.info("Running... %s", " ".join(argv))
+
+    env = augmentEnv()
+    for var in env_prune:
+        env.pop(var, None)
+
+    if reset_lang:
+        env.update({"LC_ALL": "C"})
+
+    if env_add:
+        env.update(env_add)
+
+    return subprocess.Popen(argv,
+                            stdin=stdin,
+                            stdout=stdout,
+                            stderr=stderr,
+                            close_fds=True,
+                            preexec_fn=preexec, cwd=root, env=env, **kwargs)
+
+def _run_program(argv, root='/', stdin=None, stdout=None, env_prune=None, log_output=True,
+        binary_output=False, filter_stderr=False, raise_err=False, callback=None):
+    """ Run an external program, log the output and return it to the caller
+
+        :param argv: The command to run and argument
+        :param root: The directory to chroot to before running command.
+        :param stdin: The file object to read stdin from.
+        :param stdout: Optional file object to write the output to.
+        :param env_prune: environment variable to remove before execution
+        :param log_output: whether to log the output of command
+        :param binary_output: whether to treat the output of command as binary data
+        :param filter_stderr: whether to exclude the contents of stderr from the returned output
+        :param raise_err: whether to raise a CalledProcessError if the returncode is non-zero
+        :param callback: method to call while waiting for process to finish, passed Popen object
+        :return: The return code of the command and the output
+        :raises: OSError or CalledProcessError
+    """
+    try:
+        if filter_stderr:
+            stderr = subprocess.PIPE
+        else:
+            stderr = subprocess.STDOUT
+
+        proc = startProgram(argv, root=root, stdin=stdin, stdout=subprocess.PIPE, stderr=stderr,
+                            env_prune=env_prune, universal_newlines=not binary_output)
+
+        if callback:
+            while callback(proc) and proc.poll() is None:
+                sleep(1)
+
+        (output_string, err_string) = proc.communicate()
+        if output_string:
+            if binary_output:
+                output_lines = [output_string]
+            else:
+                if output_string[-1] != "\n":
+                    output_string = output_string + "\n"
+                output_lines = output_string.splitlines(True)
+
+            if log_output:
+                with program_log_lock:
+                    for line in output_lines:
+                        program_log.info(line.strip())
+
+            if stdout:
+                stdout.write(output_string)
+
+        # If stderr was filtered, log it separately
+        if filter_stderr and err_string and log_output:
+            err_lines = err_string.splitlines(True)
+
+            with program_log_lock:
+                for line in err_lines:
+                    program_log.info(line.strip())
+
+    except OSError as e:
+        with program_log_lock:
+            program_log.error("Error running %s: %s", argv[0], e.strerror)
+        raise
+
+    with program_log_lock:
+        program_log.debug("Return code: %d", proc.returncode)
+
+    if proc.returncode and raise_err:
+        raise subprocess.CalledProcessError(proc.returncode, argv)
+
+    return (proc.returncode, output_string)
+
+[docs]def execWithRedirect(command, argv, stdin=None, stdout=None, root='/', env_prune=None,
+                     log_output=True, binary_output=False, raise_err=False, callback=None):
+    """ Run an external program and redirect the output to a file.
+
+        :param command: The command to run
+        :param argv: The argument list
+        :param stdin: The file object to read stdin from.
+        :param stdout: Optional file object to redirect stdout and stderr to.
+        :param root: The directory to chroot to before running command.
+        :param env_prune: environment variable to remove before execution
+        :param log_output: whether to log the output of command
+        :param binary_output: whether to treat the output of command as binary data
+        :param raise_err: whether to raise a CalledProcessError if the returncode is non-zero
+        :param callback: method to call while waiting for process to finish, passed Popen object
+        :return: The return code of the command
+    """
+    argv = [command] + list(argv)
+    return _run_program(argv, stdin=stdin, stdout=stdout, root=root, env_prune=env_prune,
+            log_output=log_output, binary_output=binary_output, raise_err=raise_err, callback=callback)[0]
+
+[docs]def execWithCapture(command, argv, stdin=None, root='/', log_output=True, filter_stderr=False,
+                    raise_err=False, callback=None):
+    """ Run an external program and capture standard out and err.
+
+        :param command: The command to run
+        :param argv: The argument list
+        :param stdin: The file object to read stdin from.
+        :param root: The directory to chroot to before running command.
+        :param log_output: Whether to log the output of command
+        :param filter_stderr: Whether stderr should be excluded from the returned output
+        :param raise_err: whether to raise a CalledProcessError if the returncode is non-zero
+        :return: The output of the command
+    """
+    argv = [command] + list(argv)
+    return _run_program(argv, stdin=stdin, root=root, log_output=log_output, filter_stderr=filter_stderr,
+                        raise_err=raise_err, callback=callback)[1]
+
+[docs]def runcmd(cmd, **kwargs):
+    """ run execWithRedirect with raise_err=True
+    """
+    kwargs["raise_err"] = True
+    return execWithRedirect(cmd[0], cmd[1:], **kwargs)
+
+[docs]def runcmd_output(cmd, **kwargs):
+    """ run execWithCapture with raise_err=True
+    """
+    kwargs["raise_err"] = True
+    return execWithCapture(cmd[0], cmd[1:], **kwargs)
+

+ +

+ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/_modules/pylorax/imgutils.html b/_modules/pylorax/imgutils.html new file mode 100644 index 00000000..a3ad8b6b --- /dev/null +++ b/_modules/pylorax/imgutils.html @@ -0,0 +1,605 @@ + + + + + + + + + + pylorax.imgutils — Lorax 23.8 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+ + + + +

+ + + + + + +

+ +

Source code for pylorax.imgutils

+# imgutils.py - utility functions/classes for building disk images
+#
+# Copyright (C) 2011-2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Author(s):  Will Woods <wwoods@redhat.com>
+
+import logging
+logger = logging.getLogger("pylorax.imgutils")
+
+import os, tempfile
+from os.path import join, dirname
+from subprocess import Popen, PIPE, CalledProcessError
+import sys
+import traceback
+import multiprocessing
+from time import sleep
+
+from pylorax.sysutils import cpfile
+from pylorax.executils import execWithRedirect, execWithCapture
+from pylorax.executils import runcmd, runcmd_output
+
+######## Functions for making container images (cpio, tar, squashfs) ##########
+
+[docs]def compress(command, rootdir, outfile, compression="xz", compressargs=None):
+    '''Make a compressed archive of the given rootdir.
+    command is a list of the archiver commands to run
+    compression should be "xz", "gzip", "lzma", "bzip2", or None.
+    compressargs will be used on the compression commandline.'''
+    if compression not in (None, "xz", "gzip", "lzma", "bzip2"):
+        raise ValueError("Unknown compression type %s" % compression)
+    compressargs = compressargs or ["-9"]
+    if compression == "xz":
+        compressargs.insert(0, "--check=crc32")
+    if compression is None:
+        compression = "cat" # this is a little silly
+        compressargs = []
+
+    # make compression run with multiple threads if possible
+    if compression in ("xz", "lzma"):
+        compressargs.insert(0, "-T%d" % multiprocessing.cpu_count())
+    elif compression == "gzip":
+        compression = "pigz"
+        compressargs.insert(0, "-p%d" % multiprocessing.cpu_count())
+    elif compression == "bzip2":
+        compression = "pbzip2"
+        compressargs.insert(0, "-p%d" % multiprocessing.cpu_count())
+
+    logger.debug("find %s -print0 |%s | %s %s > %s", rootdir, " ".join(command),
+                 compression, " ".join(compressargs), outfile)
+    find, archive, comp = None, None, None
+    try:
+        find = Popen(["find", ".", "-print0"], stdout=PIPE, cwd=rootdir)
+        archive = Popen(command, stdin=find.stdout, stdout=PIPE, cwd=rootdir)
+        comp = Popen([compression] + compressargs,
+                     stdin=archive.stdout, stdout=open(outfile, "wb"))
+        comp.wait()
+        return comp.returncode
+    except OSError as e:
+        logger.error(e)
+        # Kill off any hanging processes
+        list(p.kill() for p in (find, archive, comp) if p)
+        return 1
+
+[docs]def mkcpio(rootdir, outfile, compression="xz", compressargs=None):
+    compressargs = compressargs or ["-9"]
+    return compress(["cpio", "--null", "--quiet", "-H", "newc", "-o"],
+                    rootdir, outfile, compression, compressargs)
+
+[docs]def mktar(rootdir, outfile, compression="xz", compressargs=None):
+    compressargs = compressargs or ["-9"]
+    return compress(["tar", "--no-recursion", "--selinux", "--acls", "--xattrs", "-cf-", "--null", "-T-"],
+                    rootdir, outfile, compression, compressargs)
+
+[docs]def mksquashfs(rootdir, outfile, compression="default", compressargs=None):
+    '''Make a squashfs image containing the given rootdir.'''
+    compressargs = compressargs or []
+    if compression != "default":
+        compressargs = ["-comp", compression] + compressargs
+    return execWithRedirect("mksquashfs", [rootdir, outfile] + compressargs)
+
+[docs]def mkrootfsimg(rootdir, outfile, label, size=2, sysroot=""):
+    """
+    Make rootfs image from a directory
+
+    :param str rootdir: Root directory
+    :param str outfile: Path of output image file
+    :param str label: Filesystem label
+    :param int size: Size of the image in GiB, if None computed automatically
+    :param str sysroot: path to system (deployment) root relative to physical root
+    """
+    if size:
+        fssize = size * (1024*1024*1024) # 2GB sparse file compresses down to nothin'
+    else:
+        fssize = None       # Let mkext4img figure out the needed size
+
+    mkext4img(rootdir, outfile, label=label, size=fssize)
+    # Reset selinux context on new rootfs
+    with LoopDev(outfile) as loopdev:
+        with Mount(loopdev) as mnt:
+            cmd = [ "setfiles", "-e", "/proc", "-e", "/sys", "-e", "/dev", "-e", "/install",
+                    "/etc/selinux/targeted/contexts/files/file_contexts", "/"]
+            root = join(mnt, sysroot.lstrip("/"))
+            runcmd(cmd, root=root)
+
+######## Utility functions ###############################################
+
+[docs]def mksparse(outfile, size):
+    '''use os.ftruncate to create a sparse file of the given size.'''
+    fobj = open(outfile, "w")
+    os.ftruncate(fobj.fileno(), size)
+
+[docs]def mkqcow2(outfile, size, options=None):
+    '''use qemu-img to create a file of the given size.
+       options is a list of options passed to qemu-img
+
+       Default format is qcow2, override by passing "-f", fmt
+       in options.
+    '''
+    options = options or []
+    if "-f" not in options:
+        options.extend(["-f", "qcow2"])
+    runcmd(["qemu-img", "create"] + options + [outfile, str(size)])
+
+[docs]def loop_attach(outfile):
+    '''Attach a loop device to the given file. Return the loop device name.
+    Raises CalledProcessError if losetup fails.'''
+    dev = runcmd_output(["losetup", "--find", "--show", outfile])
+    return dev.strip()
+
+[docs]def loop_detach(loopdev):
+    '''Detach the given loop device. Return False on failure.'''
+    return (execWithRedirect("losetup", ["--detach", loopdev]) == 0)
+
+[docs]def get_loop_name(path):
+    '''Return the loop device associated with the path.
+    Raises RuntimeError if more than one loop is associated'''
+    buf = runcmd_output(["losetup", "-j", path])
+    if len(buf.splitlines()) > 1:
+        # there should never be more than one loop device listed
+        raise RuntimeError("multiple loops associated with %s" % path)
+    name = os.path.basename(buf.split(":")[0])
+    return name
+
+[docs]def dm_attach(dev, size, name=None):
+    '''Attach a devicemapper device to the given device, with the given size.
+    If name is None, a random name will be chosen. Returns the device name.
+    raises CalledProcessError if dmsetup fails.'''
+    if name is None:
+        name = tempfile.mktemp(prefix="lorax.imgutils.", dir="")
+    runcmd(["dmsetup", "create", name, "--table",
+                       "0 %i linear %s 0" % (size/512, dev)])
+    return name
+
+[docs]def dm_detach(dev):
+    '''Detach the named devicemapper device. Returns False if dmsetup fails.'''
+    dev = dev.replace("/dev/mapper/", "") # strip prefix, if it's there
+    return execWithRedirect("dmsetup", ["remove", dev])
+
+[docs]def mount(dev, opts="", mnt=None):
+    '''Mount the given device at the given mountpoint, using the given opts.
+    opts should be a comma-separated string of mount options.
+    if mnt is none, a temporary directory will be created and its path will be
+    returned.
+    raises CalledProcessError if mount fails.'''
+    if mnt is None:
+        mnt = tempfile.mkdtemp(prefix="lorax.imgutils.")
+        logger.debug("make tmp mountdir %s", mnt)
+    cmd = ["mount"]
+    if opts:
+        cmd += ["-o", opts]
+    cmd += [dev, mnt]
+    runcmd(cmd)
+    return mnt
+
+[docs]def umount(mnt,  lazy=False, maxretry=3, retrysleep=1.0):
+    '''Unmount the given mountpoint. If lazy is True, do a lazy umount (-l).
+    If the mount was a temporary dir created by mount, it will be deleted.
+    raises CalledProcessError if umount fails.'''
+    cmd = ["umount"]
+    if lazy: cmd += ["-l"]
+    cmd += [mnt]
+    count = 0
+    while maxretry > 0:
+        try:
+            rv = runcmd(cmd)
+        except CalledProcessError:
+            count += 1
+            if count == maxretry:
+                raise
+            logger.warn("failed to unmount %s. retrying (%d/%d)...",
+                         mnt, count, maxretry)
+            if logger.getEffectiveLevel() <= logging.DEBUG:
+                fuser = execWithCapture("fuser", ["-vm", mnt])
+                logger.debug("fuser -vm:\n%s\n", fuser)
+            sleep(retrysleep)
+        else:
+            break
+    if 'lorax.imgutils' in mnt:
+        os.rmdir(mnt)
+        logger.debug("remove tmp mountdir %s", mnt)
+    return (rv == 0)
+
+[docs]def copytree(src, dest, preserve=True):
+    '''Copy a tree of files using cp -a, thus preserving modes, timestamps,
+    links, acls, sparse files, xattrs, selinux contexts, etc.
+    If preserve is False, uses cp -R (useful for modeless filesystems)
+    raises CalledProcessError if copy fails.'''
+    logger.debug("copytree %s %s", src, dest)
+    cp = ["cp", "-a"] if preserve else ["cp", "-R", "-L"]
+    cp += [join(src, "."), os.path.abspath(dest)]
+    runcmd(cp)
+
+[docs]def do_grafts(grafts, dest, preserve=True):
+    '''Copy each of the items listed in grafts into dest.
+    If the key ends with '/' it's assumed to be a directory which should be
+    created, otherwise just the leading directories will be created.'''
+    for imgpath, filename in grafts.items():
+        if imgpath[-1] == '/':
+            targetdir = join(dest, imgpath)
+            imgpath = imgpath[:-1]
+        else:
+            targetdir = join(dest, dirname(imgpath))
+        if not os.path.isdir(targetdir):
+            os.makedirs(targetdir)
+        if os.path.isdir(filename):
+            copytree(filename, join(dest, imgpath), preserve)
+        else:
+            cpfile(filename, join(dest, imgpath))
+
+[docs]def round_to_blocks(size, blocksize):
+    '''If size isn't a multiple of blocksize, round up to the next multiple'''
+    diff = size % blocksize
+    if diff or not size:
+        size += blocksize - diff
+    return size
+
+# TODO: move filesystem data outside this function
+[docs]def estimate_size(rootdir, graft=None, fstype=None, blocksize=4096, overhead=128):
+    graft = graft or {}
+    getsize = lambda f: os.lstat(f).st_size
+    if fstype == "btrfs":
+        overhead = 64*1024 # don't worry, it's all sparse
+    if fstype == "hfsplus":
+        overhead = 200 # hack to deal with two bootloader copies
+    if fstype in ("vfat", "msdos"):
+        blocksize = 2048
+        getsize = lambda f: os.stat(f).st_size # no symlinks, count as copies
+    total = overhead*blocksize
+    dirlist = list(graft.values())
+    if rootdir:
+        dirlist.append(rootdir)
+    for root in dirlist:
+        for top, dirs, files in os.walk(root):
+            for f in files + dirs:
+                total += round_to_blocks(getsize(join(top,f)), blocksize)
+    if fstype == "btrfs":
+        total = max(256*1024*1024, total) # btrfs minimum size: 256MB
+    return total
+
+######## Execution contexts - use with the 'with' statement ##############
+
+[docs]class LoopDev(object):
+    def __init__(self, filename, size=None):
+        self.loopdev = None
+        self.filename = filename
+        if size:
+            mksparse(self.filename, size)
+    def __enter__(self):
+        self.loopdev = loop_attach(self.filename)
+        return self.loopdev
+    def __exit__(self, exc_type, exc_value, tracebk):
+        loop_detach(self.loopdev)
+
+[docs]class DMDev(object):
+    def __init__(self, dev, size, name=None):
+        self.mapperdev = None
+        (self.dev, self.size, self.name) = (dev, size, name)
+    def __enter__(self):
+        self.mapperdev = dm_attach(self.dev, self.size, self.name)
+        return self.mapperdev
+    def __exit__(self, exc_type, exc_value, tracebk):
+        dm_detach(self.mapperdev)
+
+[docs]class Mount(object):
+    def __init__(self, dev, opts="", mnt=None):
+        (self.dev, self.opts, self.mnt) = (dev, opts, mnt)
+    def __enter__(self):
+        self.mnt = mount(self.dev, self.opts, self.mnt)
+        return self.mnt
+    def __exit__(self, exc_type, exc_value, tracebk):
+        umount(self.mnt)
+
+[docs]class PartitionMount(object):
+    """ Mount a partitioned image file using kpartx """
+    def __init__(self, disk_img, mount_ok=None):
+        """
+        disk_img is the full path to a partitioned disk image
+        mount_ok is a function that is passed the mount point and
+        returns True if it should be mounted.
+        """
+        self.mount_dev = None
+        self.mount_size = None
+        self.mount_dir = None
+        self.disk_img = disk_img
+        self.mount_ok = mount_ok
+
+        # Default is to mount partition with /etc/passwd
+        if not self.mount_ok:
+            self.mount_ok = lambda mount_dir: os.path.isfile(mount_dir+"/etc/passwd")
+
+        # Example kpartx output
+        # kpartx -p p -v -a /tmp/diskV2DiCW.im
+        # add map loop2p1 (253:2): 0 3481600 linear /dev/loop2 2048
+        # add map loop2p2 (253:3): 0 614400 linear /dev/loop2 3483648
+        kpartx_output = runcmd_output(["kpartx", "-v", "-a", "-s", self.disk_img])
+        logger.debug(kpartx_output)
+
+        # list of (deviceName, sizeInBytes)
+        self.loop_devices = []
+        for line in kpartx_output.splitlines():
+            # add map loop2p3 (253:4): 0 7139328 linear /dev/loop2 528384
+            # 3rd element is size in 512 byte blocks
+            if line.startswith("add map "):
+                fields = line[8:].split()
+                self.loop_devices.append( (fields[0], int(fields[3])*512) )
+
+    def __enter__(self):
+        # Mount the device selected by mount_ok, if possible
+        mount_dir = tempfile.mkdtemp()
+        for dev, size in self.loop_devices:
+            try:
+                mount( "/dev/mapper/"+dev, mnt=mount_dir )
+                if self.mount_ok(mount_dir):
+                    self.mount_dir = mount_dir
+                    self.mount_dev = dev
+                    self.mount_size = size
+                    break
+                umount( mount_dir )
+            except CalledProcessError:
+                logger.debug(traceback.format_exc())
+        if self.mount_dir:
+            logger.info("Partition mounted on %s size=%s", self.mount_dir, self.mount_size)
+        else:
+            logger.debug("Unable to mount anything from %s", self.disk_img)
+            os.rmdir(mount_dir)
+        return self
+
+    def __exit__(self, exc_type, exc_value, tracebk):
+        if self.mount_dir:
+            umount( self.mount_dir )
+            os.rmdir(self.mount_dir)
+            self.mount_dir = None
+        execWithRedirect("kpartx", ["-d", "-s", self.disk_img])
+
+
+######## Functions for making filesystem images ##########################
+
+[docs]def mkfsimage(fstype, rootdir, outfile, size=None, mkfsargs=None, mountargs="", graft=None):
+    '''Generic filesystem image creation function.
+    fstype should be a filesystem type - "mkfs.${fstype}" must exist.
+    graft should be a dict: {"some/path/in/image": "local/file/or/dir"};
+      if the path ends with a '/' it's assumed to be a directory.
+    Will raise CalledProcessError if something goes wrong.'''
+    mkfsargs = mkfsargs or []
+    graft = graft or {}
+    preserve = (fstype not in ("msdos", "vfat"))
+    if not size:
+        size = estimate_size(rootdir, graft, fstype)
+    with LoopDev(outfile, size) as loopdev:
+        try:
+            runcmd(["mkfs.%s" % fstype] + mkfsargs + [loopdev])
+        except CalledProcessError as e:
+            logger.error("mkfs exited with a non-zero return code: %d", e.returncode)
+            logger.error(e.output)
+            sys.exit(e.returncode)
+
+        with Mount(loopdev, mountargs) as mnt:
+            if rootdir:
+                copytree(rootdir, mnt, preserve)
+            do_grafts(graft, mnt, preserve)
+
+    # Make absolutely sure that the data has been written
+    runcmd(["sync"])
+
+# convenience functions with useful defaults
+[docs]def mkdosimg(rootdir, outfile, size=None, label="", mountargs="shortname=winnt,umask=0077", graft=None):
+    graft = graft or {}
+    mkfsimage("msdos", rootdir, outfile, size, mountargs=mountargs,
+              mkfsargs=["-n", label], graft=graft)
+
+[docs]def mkext4img(rootdir, outfile, size=None, label="", mountargs="", graft=None):
+    graft = graft or {}
+    mkfsimage("ext4", rootdir, outfile, size, mountargs=mountargs,
+              mkfsargs=["-L", label, "-b", "1024", "-m", "0"], graft=graft)
+
+[docs]def mkbtrfsimg(rootdir, outfile, size=None, label="", mountargs="", graft=None):
+    graft = graft or {}
+    mkfsimage("btrfs", rootdir, outfile, size, mountargs=mountargs,
+               mkfsargs=["-L", label], graft=graft)
+
+[docs]def mkhfsimg(rootdir, outfile, size=None, label="", mountargs="", graft=None):
+    graft = graft or {}
+    mkfsimage("hfsplus", rootdir, outfile, size, mountargs=mountargs,
+              mkfsargs=["-v", label], graft=graft)
+

+ +

+ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/_modules/pylorax/ltmpl.html b/_modules/pylorax/ltmpl.html new file mode 100644 index 00000000..30423537 --- /dev/null +++ b/_modules/pylorax/ltmpl.html @@ -0,0 +1,902 @@ + + + + + + + + + + pylorax.ltmpl — Lorax 23.8 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+ + + + +

+ + + + + + +

+ +

Source code for pylorax.ltmpl

+#
+# ltmpl.py
+#
+# Copyright (C) 2009-2015  Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Red Hat Author(s):  Martin Gracik <mgracik@redhat.com>
+#                     Will Woods <wwoods@redhat.com>
+#
+
+import logging
+logger = logging.getLogger("pylorax.ltmpl")
+
+import os, re, glob, shlex, fnmatch
+from os.path import basename, isdir
+from subprocess import CalledProcessError
+
+from pylorax.sysutils import joinpaths, cpfile, mvfile, replace, remove
+from pylorax.dnfhelper import LoraxDownloadCallback, LoraxRpmCallback
+from pylorax.base import DataHolder
+from pylorax.executils import runcmd, runcmd_output
+from pylorax.imgutils import mkcpio
+import pylorax.output as output
+
+from mako.lookup import TemplateLookup
+from mako.exceptions import text_error_template
+import sys, traceback
+import struct
+import dnf
+import multiprocessing
+import queue
+import collections
+
+[docs]class LoraxTemplate(object):
+    def __init__(self, directories=None):
+        directories = directories or ["/usr/share/lorax"]
+        # we have to add ["/"] to the template lookup directories or the
+        # file includes won't work properly for absolute paths
+        self.directories = ["/"] + directories
+
+[docs]    def parse(self, template_file, variables):
+        lookup = TemplateLookup(directories=self.directories)
+        template = lookup.get_template(template_file)
+
+        try:
+            textbuf = template.render(**variables)
+        except:
+            logger.error("Problem rendering %s (%s):", template_file, variables)
+            logger.error(text_error_template().render())
+            raise
+
+        # split, strip and remove empty lines
+        lines = textbuf.splitlines()
+        lines = [line.strip() for line in lines]
+        lines = [line for line in lines if line]
+
+        # remove comments
+        lines = [line for line in lines if not line.startswith("#")]
+
+        # split with shlex and perform brace expansion
+        lines = [split_and_expand(line) for line in lines]
+
+        return lines
+
+[docs]def split_and_expand(line):
+    return [exp for word in shlex.split(line) for exp in brace_expand(word)]
+
+[docs]def brace_expand(s):
+    if not ('{' in s and ',' in s and '}' in s):
+        yield s
+    else:
+        right = s.find('}')
+        left = s[:right].rfind('{')
+        (prefix, choices, suffix) = (s[:left], s[left+1:right], s[right+1:])
+        for choice in choices.split(','):
+            for alt in brace_expand(prefix+choice+suffix):
+                yield alt
+
+[docs]def rglob(pathname, root="/", fatal=False):
+    seen = set()
+    rootlen = len(root)+1
+    for f in glob.iglob(joinpaths(root, pathname)):
+        if f not in seen:
+            seen.add(f)
+            yield f[rootlen:] # remove the root to produce relative path
+    if fatal and not seen:
+        raise IOError("nothing matching %s in %s" % (pathname, root))
+
+[docs]def rexists(pathname, root=""):
+    # Generator is always True, even with no values;
+    # bool(rglob(...)) won't work here.
+    for _path in rglob(pathname, root):
+        return True
+    return False
+
+# TODO: operate inside an actual chroot for safety? Not that RPM bothers..
+[docs]class LoraxTemplateRunner(object):
+    '''
+    This class parses and executes Lorax templates. Sample usage:
+
+      # install a bunch of packages
+      runner = LoraxTemplateRunner(inroot=rundir, outroot=rundir, dbo=dnf_obj)
+      runner.run("install-packages.ltmpl")
+
+      # modify a runtime dir
+      runner = LoraxTemplateRunner(inroot=rundir, outroot=newrun)
+      runner.run("runtime-transmogrify.ltmpl")
+
+    NOTES:
+
+    * Parsing procedure is roughly:
+      1. Mako template expansion (on the whole file)
+      2. For each line of the result,
+        a. Whitespace splitting (using shlex.split())
+        b. Brace expansion (using brace_expand())
+        c. If the first token is the name of a function, call that function
+           with the rest of the line as arguments
+
+    * Parsing and execution are *separate* passes - so you can't use the result
+      of a command in an %if statement (or any other control statements)!
+
+    * Commands that run external programs (systemctl, gconfset) currently use
+      the *host*'s copy of that program, which may cause problems if there's a
+      big enough difference between the host and the image you're modifying.
+
+    * The commands are not executed under a real chroot, so absolute symlinks
+      will point *outside* the inroot/outroot. Be careful with symlinks!
+
+    ADDING NEW COMMANDS:
+
+    * Each template command is just a method of the LoraxTemplateRunner
+      object - so adding a new command is as easy as adding a new function.
+
+    * Each function gets arguments that correspond to the rest of the tokens
+      on that line (after word splitting and brace expansion)
+
+    * Commands should raise exceptions for errors - don't use sys.exit()
+    '''
+    def __init__(self, inroot, outroot, dbo=None, fatalerrors=True,
+                                        templatedir=None, defaults=None):
+        self.inroot = inroot
+        self.outroot = outroot
+        self.dbo = dbo
+        self.fatalerrors = fatalerrors
+        self.templatedir = templatedir or "/usr/share/lorax"
+        self.templatefile = None
+        # some builtin methods
+        self.builtins = DataHolder(exists=lambda p: rexists(p, root=inroot),
+                                   glob=lambda g: list(rglob(g, root=inroot)))
+        self.defaults = defaults or {}
+        self.results = DataHolder(treeinfo=dict()) # just treeinfo for now
+        # TODO: set up custom logger with a filter to add line info
+
+    def _out(self, path):
+        return joinpaths(self.outroot, path)
+    def _in(self, path):
+        return joinpaths(self.inroot, path)
+
+    def _filelist(self, *pkgs):
+        """ Return the list of files in the packages """
+        pkglist = []
+        for pkg_glob in pkgs:
+            pkglist += list(self.dbo.sack.query().installed().filter(name__glob=pkg_glob))
+
+        # dnf/hawkey doesn't make any distinction between file, dir or ghost like yum did
+        # so only return the files.
+        return set(f for pkg in pkglist for f in pkg.files if not os.path.isdir(self._out(f)))
+
+    def _getsize(self, *files):
+        return sum(os.path.getsize(self._out(f)) for f in files if os.path.isfile(self._out(f)))
+
+[docs]    def run(self, templatefile, **variables):
+        for k,v in list(self.defaults.items()) + list(self.builtins.items()):
+            variables.setdefault(k,v)
+        logger.debug("executing %s with variables=%s", templatefile, variables)
+        self.templatefile = templatefile
+        t = LoraxTemplate(directories=[self.templatedir])
+        commands = t.parse(templatefile, variables)
+        self._run(commands)
+
+
+    def _run(self, parsed_template):
+        logger.info("running %s", self.templatefile)
+        for (num, line) in enumerate(parsed_template,1):
+            logger.debug("template line %i: %s", num, " ".join(line))
+            skiperror = False
+            (cmd, args) = (line[0], line[1:])
+            # Following Makefile convention, if the command is prefixed with
+            # a dash ('-'), we'll ignore any errors on that line.
+            if cmd.startswith('-'):
+                cmd = cmd[1:]
+                skiperror = True
+            try:
+                # grab the method named in cmd and pass it the given arguments
+                f = getattr(self, cmd, None)
+                if cmd[0] == '_' or cmd == 'run' or not isinstance(f, collections.Callable):
+                    raise ValueError("unknown command %s" % cmd)
+                f(*args)
+            except Exception: # pylint: disable=broad-except
+                if skiperror:
+                    logger.debug("ignoring error")
+                    continue
+                logger.error("template command error in %s:", self.templatefile)
+                logger.error("  %s", " ".join(line))
+                # format the exception traceback
+                exclines = traceback.format_exception(*sys.exc_info())
+                # skip the bit about "ltmpl.py, in _run()" - we know that
+                exclines.pop(1)
+                # log the "ErrorType: this is what happened" line
+                logger.error("  " + exclines[-1].strip())
+                # and log the entire traceback to the debug log
+                for line in ''.join(exclines).splitlines():
+                    logger.debug("  " + line)
+                if self.fatalerrors:
+                    raise
+
+[docs]    def install(self, srcglob, dest):
+        '''
+        install SRC DEST
+          Copy the given file (or files, if a glob is used) from the input
+          tree to the given destination in the output tree.
+          The path to DEST must exist in the output tree.
+          If DEST is a directory, SRC will be copied into that directory.
+          If DEST doesn't exist, SRC will be copied to a file with that name,
+          assuming the rest of the path exists.
+          This is pretty much like how the 'cp' command works.
+          Examples:
+            install usr/share/myconfig/grub.conf /boot
+            install /usr/share/myconfig/grub.conf.in /boot/grub.conf
+        '''
+        for src in rglob(self._in(srcglob), fatal=True):
+            cpfile(src, self._out(dest))
+
+[docs]    def installimg(self, srcdir, destfile):
+        '''
+        installimg SRCDIR DESTFILE
+          Create a compressed cpio archive of the contents of SRCDIR and place
+          it in DESTFILE.
+
+          If SRCDIR doesn't exist or is empty nothing is created.
+
+          Examples:
+            installimg ${LORAXDIR}/product/ images/product.img
+            installimg ${LORAXDIR}/updates/ images/updates.img
+        '''
+        if not os.path.isdir(self._in(srcdir)) or not os.listdir(self._in(srcdir)):
+            return
+        logger.info("Creating image file %s from contents of %s", self._out(destfile), self._in(srcdir))
+        mkcpio(self._in(srcdir), self._out(destfile))
+
+[docs]    def mkdir(self, *dirs):
+        '''
+        mkdir DIR [DIR ...]
+          Create the named DIR(s). Will create leading directories as needed.
+          Example:
+            mkdir /images
+        '''
+        for d in dirs:
+            d = self._out(d)
+            if not isdir(d):
+                os.makedirs(d)
+
+[docs]    def replace(self, pat, repl, *fileglobs):
+        '''
+        replace PATTERN REPLACEMENT FILEGLOB [FILEGLOB ...]
+          Find-and-replace the given PATTERN (Python-style regex) with the given
+          REPLACEMENT string for each of the files listed.
+          Example:
+            replace @VERSION@ ${product.version} /boot/grub.conf /boot/isolinux.cfg
+        '''
+        match = False
+        for g in fileglobs:
+            for f in rglob(self._out(g)):
+                match = True
+                replace(f, pat, repl)
+        if not match:
+            raise IOError("no files matched %s" % " ".join(fileglobs))
+
+[docs]    def append(self, filename, data):
+        '''
+        append FILE STRING
+          Append STRING (followed by a newline character) to FILE.
+          Python character escape sequences ('\\n', '\\t', etc.) will be
+          converted to the appropriate characters.
+          Examples:
+            append /etc/depmod.d/dd.conf "search updates built-in"
+            append /etc/resolv.conf ""
+        '''
+        with open(self._out(filename), "a") as fobj:
+            fobj.write(bytes(data, "utf8").decode('unicode_escape')+"\n")
+
+[docs]    def treeinfo(self, section, key, *valuetoks):
+        '''
+        treeinfo SECTION KEY ARG [ARG ...]
+          Add an item to the treeinfo data store.
+          The given SECTION will have a new item added where
+          KEY = ARG ARG ...
+          Example:
+            treeinfo images-${kernel.arch} boot.iso images/boot.iso
+        '''
+        if section not in self.results.treeinfo:
+            self.results.treeinfo[section] = dict()
+        self.results.treeinfo[section][key] = " ".join(valuetoks)
+
+[docs]    def installkernel(self, section, src, dest):
+        '''
+        installkernel SECTION SRC DEST
+          Install the kernel from SRC in the input tree to DEST in the output
+          tree, and then add an item to the treeinfo data store, in the named
+          SECTION, where "kernel" = DEST.
+
+          Equivalent to:
+            install SRC DEST
+            treeinfo SECTION kernel DEST
+        '''
+        self.install(src, dest)
+        self.treeinfo(section, "kernel", dest)
+
+[docs]    def installinitrd(self, section, src, dest):
+        '''
+        installinitrd SECTION SRC DEST
+          Same as installkernel, but for "initrd".
+        '''
+        self.install(src, dest)
+        self.chmod(dest, '644')
+        self.treeinfo(section, "initrd", dest)
+
+[docs]    def installupgradeinitrd(self, section, src, dest):
+        '''
+        installupgradeinitrd SECTION SRC DEST
+          Same as installkernel, but for "upgrade".
+        '''
+        self.install(src, dest)
+        self.chmod(dest, '644')
+        self.treeinfo(section, "upgrade", dest)
+
+[docs]    def hardlink(self, src, dest):
+        '''
+        hardlink SRC DEST
+          Create a hardlink at DEST which is linked to SRC.
+        '''
+        if isdir(self._out(dest)):
+            dest = joinpaths(dest, basename(src))
+        os.link(self._out(src), self._out(dest))
+
+[docs]    def symlink(self, target, dest):
+        '''
+        symlink SRC DEST
+          Create a symlink at DEST which points to SRC.
+        '''
+        if rexists(self._out(dest)):
+            self.remove(dest)
+        os.symlink(target, self._out(dest))
+
+[docs]    def copy(self, src, dest):
+        '''
+        copy SRC DEST
+          Copy SRC to DEST.
+          If DEST is a directory, SRC will be copied inside it.
+          If DEST doesn't exist, SRC will be copied to a file with
+          that name, if the path leading to it exists.
+        '''
+        cpfile(self._out(src), self._out(dest))
+
+[docs]    def move(self, src, dest):
+        '''
+        move SRC DEST
+          Move SRC to DEST.
+        '''
+        mvfile(self._out(src), self._out(dest))
+
+[docs]    def remove(self, *fileglobs):
+        '''
+        remove FILEGLOB [FILEGLOB ...]
+          Remove all the named files or directories.
+          Will *not* raise exceptions if the file(s) are not found.
+        '''
+        for g in fileglobs:
+            for f in rglob(self._out(g)):
+                remove(f)
+                logger.debug("removed %s", f)
+
+[docs]    def chmod(self, fileglob, mode):
+        '''
+        chmod FILEGLOB OCTALMODE
+          Change the mode of all the files matching FILEGLOB to OCTALMODE.
+        '''
+        for f in rglob(self._out(fileglob), fatal=True):
+            os.chmod(f, int(mode,8))
+
+    # TODO: do we need a new command for gsettings?
+[docs]    def gconfset(self, path, keytype, value, outfile=None):
+        '''
+        gconfset PATH KEYTYPE VALUE [OUTFILE]
+          Set the given gconf PATH, with type KEYTYPE, to the given value.
+          OUTFILE defaults to /etc/gconf/gconf.xml.defaults if not given.
+          Example:
+            gconfset /apps/metacity/general/num_workspaces int 1
+        '''
+        if outfile is None:
+            outfile = self._out("etc/gconf/gconf.xml.defaults")
+        cmd = ["gconftool-2", "--direct",
+                    "--config-source=xml:readwrite:%s" % outfile,
+                    "--set", "--type", keytype, path, value]
+        runcmd(cmd)
+
+[docs]    def log(self, msg):
+        '''
+        log MESSAGE
+          Emit the given log message. Be sure to put it in quotes!
+          Example:
+            log "Reticulating splines, please wait..."
+        '''
+        logger.info(msg)
+
+    # TODO: add ssh-keygen, mkisofs(?), find, and other useful commands
+[docs]    def runcmd(self, *cmdlist):
+        '''
+        runcmd CMD [ARG ...]
+          Run the given command with the given arguments.
+
+          NOTE: All paths given MUST be COMPLETE, ABSOLUTE PATHS to the file
+          or files mentioned. ${root}/${inroot}/${outroot} are good for
+          constructing these paths.
+
+          FURTHER NOTE: Please use this command only as a last resort!
+          Whenever possible, you should use the existing template commands.
+          If the existing commands don't do what you need, fix them!
+
+          Examples:
+            (this should be replaced with a "find" function)
+            runcmd find ${root} -name "*.pyo" -type f -delete
+            %for f in find(root, name="*.pyo"):
+                remove ${f}
+            %endfor
+        '''
+        cmd = cmdlist
+        logger.debug('running command: %s', cmd)
+        if cmd[0].startswith("--chdir="):
+            logger.error("--chdir is no longer supported for runcmd.")
+            raise ValueError("--chdir is no longer supported for runcmd.")
+
+        try:
+            stdout = runcmd_output(cmd)
+            if stdout:
+                logger.debug('command output:\n%s', stdout)
+            logger.debug("command finished successfully")
+        except CalledProcessError as e:
+            if e.output:
+                logger.debug('command output:\n%s', e.output)
+            logger.debug('command returned failure (%d)', e.returncode)
+            raise
+
+[docs]    def installpkg(self, *pkgs):
+        '''
+        installpkg [--required] PKGGLOB [PKGGLOB ...]
+          Request installation of all packages matching the given globs.
+          Note that this is just a *request* - nothing is *actually* installed
+          until the 'run_pkg_transaction' command is given.
+        '''
+        required = False
+        if pkgs[0] == '--required':
+            pkgs = pkgs[1:]
+            required = True
+
+        for p in pkgs:
+            try:
+                self.dbo.install(p)
+            except Exception as e: # pylint: disable=broad-except
+                # FIXME: save exception and re-raise after the loop finishes
+                logger.error("installpkg %s failed: %s", p, str(e))
+                if required:
+                    raise
+
+[docs]    def removepkg(self, *pkgs):
+        '''
+        removepkg PKGGLOB [PKGGLOB...]
+          Delete the named package(s).
+          IMPLEMENTATION NOTES:
+            RPM scriptlets (%preun/%postun) are *not* run.
+            Files are deleted, but directories are left behind.
+        '''
+        for p in pkgs:
+            filepaths = [f.lstrip('/') for f in self._filelist(p)]
+            # TODO: also remove directories that aren't owned by anything else
+            if filepaths:
+                logger.debug("removepkg %s: %ikb", p, self._getsize(*filepaths)/1024)
+                self.remove(*filepaths)
+            else:
+                logger.debug("removepkg %s: no files to remove!", p)
+
+[docs]    def get_token_checked(self, process, token_queue):
+        """Try to get token from queue checking that process is still alive"""
+
+        try:
+            # wait at most a minute for the token
+            (token, msg) = token_queue.get(timeout=60)
+        except queue.Empty:
+            if process.is_alive():
+                try:
+                    # process still alive, give it 2 minutes more
+                    (token, msg) = token_queue.get(timeout=120)
+                except queue.Empty:
+                    # waited for 3 minutes and got nothing
+                    raise Exception("The transaction process got stuck somewhere (no message from it in 3 minutes)")
+            else:
+                raise Exception("The transaction process has ended abruptly")
+
+        return (token, msg)
+
+[docs]    def run_pkg_transaction(self):
+        '''
+        run_pkg_transaction
+          Actually install all the packages requested by previous 'installpkg'
+          commands.
+        '''
+
+        def do_transaction(base, token_queue):
+            try:
+                display = LoraxRpmCallback(token_queue)
+                base.do_transaction(display=display)
+            except BaseException as e:
+                logger.error("The transaction process has ended abruptly: %s", e)
+                token_queue.put(('quit', str(e)))
+
+        try:
+            logger.info("Checking dependencies")
+            self.dbo.resolve()
+        except dnf.exceptions.DepsolveError as e:
+            logger.error("Dependency check failed: %s", e)
+            raise
+        logger.info("%d packages selected", len(self.dbo.transaction))
+        if len(self.dbo.transaction) == 0:
+            raise Exception("No packages in transaction")
+
+        pkgs_to_download = self.dbo.transaction.install_set
+        logger.info("Downloading packages")
+        progress = LoraxDownloadCallback()
+        try:
+            self.dbo.download_packages(pkgs_to_download, progress)
+        except dnf.exceptions.DownloadError as e:
+            logger.error("Failed to download the following packages: %s", e)
+            raise
+
+        logger.info("Preparing transaction from installation source")
+        token_queue = multiprocessing.Queue()
+        msgout = output.LoraxOutput()
+        process = multiprocessing.Process(target=do_transaction, args=(self.dbo, token_queue))
+        process.start()
+        (token, msg) = self.get_token_checked(process, token_queue)
+
+        while token not in ('post', 'quit'):
+            if token == 'install':
+                logging.info("%s", msg)
+                msgout.writeline(msg)
+            (token, msg) = self.get_token_checked(process, token_queue)
+
+        if token == 'quit':
+            logger.error("Transaction failed.")
+            raise Exception("Transaction failed")
+
+        logger.info("Performing post-installation setup tasks")
+        process.join()
+
+        # Reset the package sack to pick up the installed packages
+        self.dbo.reset(repos=False)
+        self.dbo.fill_sack(load_system_repo=True, load_available_repos=False)
+
+        # At this point dnf should know about the installed files. Double check that it really does.
+        if len(self._filelist("anaconda-core")) == 0:
+            raise Exception("Failed to reset dbo to installed package set")
+
+[docs]    def removefrom(self, pkg, *globs):
+        '''
+        removefrom PKGGLOB [--allbut] FILEGLOB [FILEGLOB...]
+          Remove all files matching the given file globs from the package
+          (or packages) named.
+          If '--allbut' is used, all the files from the given package(s) will
+          be removed *except* the ones which match the file globs.
+          Examples:
+            removefrom usbutils /usr/bin/*
+            removefrom xfsprogs --allbut /sbin/*
+        '''
+        cmd = "%s %s" % (pkg, " ".join(globs)) # save for later logging
+        keepmatches = False
+        if globs[0] == '--allbut':
+            keepmatches = True
+            globs = globs[1:]
+        # get pkg filelist and find files that match the globs
+        filelist = self._filelist(pkg)
+        matches = set()
+        for g in globs:
+            globs_re = re.compile(fnmatch.translate(g))
+            m = [f for f in filelist if globs_re.match(f)]
+            if m:
+                matches.update(m)
+            else:
+                logger.debug("removefrom %s %s: no files matched!", pkg, g)
+        # are we removing the matches, or keeping only the matches?
+        if keepmatches:
+            remove_files = filelist.difference(matches)
+        else:
+            remove_files = matches
+        # remove the files
+        if remove_files:
+            logger.debug("removefrom %s: removed %i/%i files, %ikb/%ikb", cmd,
+                             len(remove_files), len(filelist),
+                             self._getsize(*remove_files)/1024, self._getsize(*filelist)/1024)
+            self.remove(*remove_files)
+        else:
+            logger.debug("removefrom %s: no files to remove!", cmd)
+
+[docs]    def removekmod(self, *globs):
+        '''
+        removekmod GLOB [GLOB...] [--allbut] KEEPGLOB [KEEPGLOB...]
+          Remove all files and directories matching the given file globs from the kernel
+          modules directory.
+
+          If '--allbut' is used, all the files from the modules will be removed *except*
+          the ones which match the file globs. There must be at least one initial GLOB
+          to search and one KEEPGLOB to keep. The KEEPGLOB is expanded to be *KEEPGLOB*
+          so that it will match anywhere in the path.
+
+          This only removes files from under /lib/modules/*/kernel/
+
+          Examples:
+            removekmod sound drivers/media drivers/hwmon drivers/video
+            removekmod drivers/char --allbut virtio_console hw_random
+        '''
+        cmd = " ".join(globs)
+        if "--allbut" in globs:
+            idx = globs.index("--allbut")
+            if idx == 0:
+                raise ValueError("removekmod needs at least one GLOB before --allbut")
+
+            # Apply keepglobs anywhere they appear in the path
+            keepglobs = globs[idx+1:]
+            if len(keepglobs) == 0:
+                raise ValueError("removekmod needs at least one GLOB after --allbut")
+
+            globs = globs[:idx]
+        else:
+            # Nothing to keep
+            keepglobs = []
+
+        filelist = set()
+        for g in globs:
+            for top_dir in rglob(self._out("/lib/modules/*/kernel/"+g)):
+                for root, _dirs, files in os.walk(top_dir):
+                    filelist.update(root+"/"+f for f in files)
+
+        # Remove anything matching keepglobs from the list
+        matches = set()
+        for g in keepglobs:
+            globs_re = re.compile(fnmatch.translate("*"+g+"*"))
+            m = [f for f in filelist if globs_re.match(f)]
+            if m:
+                matches.update(m)
+            else:
+                logger.debug("removekmod %s: no files matched!", g)
+        remove_files = filelist.difference(matches)
+
+        if remove_files:
+            logger.debug("removekmod: removing %d files", len(remove_files))
+            list(remove(f) for f in remove_files)
+        else:
+            logger.debug("removekmod %s: no files to remove!", cmd)
+
+[docs]    def createaddrsize(self, addr, src, dest):
+        '''
+        createaddrsize INITRD_ADDRESS INITRD ADDRSIZE
+          Create the initrd.addrsize file required in LPAR boot process.
+          Examples:
+            createaddrsize ${INITRD_ADDRESS} ${outroot}/${BOOTDIR}/initrd.img ${outroot}/${BOOTDIR}/initrd.addrsize
+        '''
+        addrsize = open(dest, "wb")
+        addrsize_data = struct.pack(">iiii", 0, int(addr, 16), 0, os.stat(src).st_size)
+        addrsize.write(addrsize_data)
+        addrsize.close()
+
+[docs]    def systemctl(self, cmd, *units):
+        '''
+        systemctl [enable|disable|mask] UNIT [UNIT...]
+          Enable, disable, or mask the given systemd units.
+          Examples:
+            systemctl disable lvm2-monitor.service
+            systemctl mask fedora-storage-init.service fedora-configure.service
+        '''
+        if cmd not in ('enable', 'disable', 'mask'):
+            raise ValueError('unsupported systemctl cmd: %s' % cmd)
+        if not units:
+            logger.debug("systemctl: no units given for %s, ignoring", cmd)
+            return
+        self.mkdir("/run/systemd/system") # XXX workaround for systemctl bug
+        systemctl = ('systemctl', '--root', self.outroot, '--no-reload',
+                     '--quiet', cmd)
+        # XXX for some reason 'systemctl enable/disable' always returns 1
+        try:
+            cmd = systemctl + units
+            runcmd(cmd)
+        except CalledProcessError:
+            pass
+

+ +

+ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/_modules/pylorax/sysutils.html b/_modules/pylorax/sysutils.html new file mode 100644 index 00000000..756b87b0 --- /dev/null +++ b/_modules/pylorax/sysutils.html @@ -0,0 +1,295 @@ + + + + + + + + + + pylorax.sysutils — Lorax 23.8 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+ + + + +

+ + + + + + +

+ +

Source code for pylorax.sysutils

+#
+# sysutils.py
+#
+# Copyright (C) 2009-2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Red Hat Author(s):  Martin Gracik <mgracik@redhat.com>
+#
+
+__all__ = ["joinpaths", "touch", "replace", "chown_", "chmod_", "remove",
+           "linktree"]
+
+import sys
+import os
+import re
+import fileinput
+import pwd
+import grp
+import glob
+import shutil
+
+from pylorax.executils import runcmd
+
+[docs]def joinpaths(*args, **kwargs):
+    path = os.path.sep.join(args)
+
+    if kwargs.get("follow_symlinks"):
+        return os.path.realpath(path)
+    else:
+        return path
+
+
+[docs]def touch(fname):
+    open(fname, "w")
+
+
+[docs]def replace(fname, find, sub):
+    fin = fileinput.input(fname, inplace=1)
+    pattern = re.compile(find)
+
+    for line in fin:
+        line = pattern.sub(sub, line)
+        sys.stdout.write(line)
+
+    fin.close()
+
+
+[docs]def chown_(path, user=None, group=None, recursive=False):
+    uid = gid = -1
+
+    if user is not None:
+        uid = pwd.getpwnam(user)[2]
+    if group is not None:
+        gid = grp.getgrnam(group)[2]
+
+    for fname in glob.iglob(path):
+        os.chown(fname, uid, gid)
+
+        if recursive and os.path.isdir(fname):
+            for nested in os.listdir(fname):
+                nested = joinpaths(fname, nested)
+                chown_(nested, user, group, recursive)
+
+
+[docs]def chmod_(path, mode, recursive=False):
+    for fname in glob.iglob(path):
+        os.chmod(fname, mode)
+
+        if recursive and os.path.isdir(fname):
+            for nested in os.listdir(fname):
+                nested = joinpaths(fname, nested)
+                chmod_(nested, mode, recursive)
+
+
+def cpfile(src, dst):
+    shutil.copy2(src, dst)
+    if os.path.isdir(dst):
+        dst = joinpaths(dst, os.path.basename(src))
+
+    return dst
+
+def mvfile(src, dst):
+    if os.path.isdir(dst):
+        dst = joinpaths(dst, os.path.basename(src))
+    os.rename(src, dst)
+    return dst
+
+[docs]def remove(target):
+    if os.path.isdir(target) and not os.path.islink(target):
+        shutil.rmtree(target)
+    else:
+        os.unlink(target)
+
+[docs]def linktree(src, dst):
+    runcmd(["/bin/cp", "-alx", src, dst])
+

+ +

+ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/_modules/pylorax/treebuilder.html b/_modules/pylorax/treebuilder.html new file mode 100644 index 00000000..9560754f --- /dev/null +++ b/_modules/pylorax/treebuilder.html @@ -0,0 +1,527 @@ + + + + + + + + + + pylorax.treebuilder — Lorax 23.8 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+ + + + +

+ + + + + + +

+ +

Source code for pylorax.treebuilder

+# treebuilder.py - handle arch-specific tree building stuff using templates
+#
+# Copyright (C) 2011-2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Author(s):  Will Woods <wwoods@redhat.com>
+
+import logging
+logger = logging.getLogger("pylorax.treebuilder")
+
+import os, re
+from os.path import basename
+from shutil import copytree, copy2
+
+from pylorax.sysutils import joinpaths, remove
+from pylorax.base import DataHolder
+from pylorax.ltmpl import LoraxTemplateRunner
+import pylorax.imgutils as imgutils
+from pylorax.executils import runcmd, runcmd_output
+
+templatemap = {
+    'i386':    'x86.tmpl',
+    'x86_64':  'x86.tmpl',
+    'ppc':     'ppc.tmpl',
+    'ppc64':   'ppc.tmpl',
+    'ppc64le': 'ppc64le.tmpl',
+    's390':    's390.tmpl',
+    's390x':   's390.tmpl',
+    'aarch64': 'aarch64.tmpl',
+    'arm':     'arm.tmpl',
+    'armhfp':  'arm.tmpl',
+}
+
+[docs]def generate_module_info(moddir, outfile=None):
+    def module_desc(mod):
+        output = runcmd_output(["modinfo", "-F", "description", mod])
+        return output.strip()
+    def read_module_set(name):
+        return set(l.strip() for l in open(joinpaths(moddir,name)) if ".ko" in l)
+    modsets = {'scsi':read_module_set("modules.block"),
+               'eth':read_module_set("modules.networking")}
+
+    modinfo = list()
+    for root, _dirs, files in os.walk(moddir):
+        for modtype, modset in modsets.items():
+            for mod in modset.intersection(files):  # modules in this dir
+                (name, _ext) = os.path.splitext(mod) # foo.ko -> (foo, .ko)
+                desc = module_desc(joinpaths(root,mod)) or "%s driver" % name
+                modinfo.append(dict(name=name, type=modtype, desc=desc))
+
+    out = open(outfile or joinpaths(moddir,"module-info"), "w")
+    out.write("Version 0\n")
+    for mod in sorted(modinfo, key=lambda m: m.get('name')):
+        out.write('{name}\n\t{type}\n\t"{desc:.65}"\n'.format(**mod))
+
+[docs]class RuntimeBuilder(object):
+    '''Builds the anaconda runtime image.'''
+    def __init__(self, product, arch, dbo, templatedir=None,
+                 installpkgs=None,
+                 add_templates=None,
+                 add_template_vars=None):
+        root = dbo.conf.installroot
+        # use a copy of product so we can modify it locally
+        product = product.copy()
+        product.name = product.name.lower()
+        self.vars = DataHolder(arch=arch, product=product, dbo=dbo, root=root,
+                               basearch=arch.basearch, libdir=arch.libdir)
+        self.dbo = dbo
+        self._runner = LoraxTemplateRunner(inroot=root, outroot=root,
+                                           dbo=dbo, templatedir=templatedir)
+        self.add_templates = add_templates or []
+        self.add_template_vars = add_template_vars or {}
+        self._installpkgs = installpkgs or []
+        self._runner.defaults = self.vars
+        self.dbo.reset()
+
+    def _install_branding(self):
+        release = None
+        q = self.dbo.sack.query()
+        a = q.available()
+        for pkg in a.filter(provides='/etc/system-release'):
+            if pkg.name.startswith('generic'):
+                continue
+            else:
+                release = pkg.name
+                break
+
+        if not release:
+            logger.error('could not get the release')
+            return
+
+        # release
+        logger.info('got release: %s', release)
+        self._runner.installpkg(release)
+
+        # logos
+        release, _suffix = release.split('-', 1)
+        self._runner.installpkg('%s-logos' % release)
+
+[docs]    def install(self):
+        '''Install packages and do initial setup with runtime-install.tmpl'''
+        self._install_branding()
+        if len(self._installpkgs) > 0:
+            self._runner.installpkg(*self._installpkgs)
+        self._runner.run("runtime-install.tmpl")
+        for tmpl in self.add_templates:
+            self._runner.run(tmpl, **self.add_template_vars)
+
+[docs]    def writepkglists(self, pkglistdir):
+        '''debugging data: write out lists of package contents'''
+        if not os.path.isdir(pkglistdir):
+            os.makedirs(pkglistdir)
+        q = self.dbo.sack.query()
+        for pkgobj in q.installed():
+            with open(joinpaths(pkglistdir, pkgobj.name), "w") as fobj:
+                for fname in pkgobj.files:
+                    fobj.write("{0}\n".format(fname))
+
+[docs]    def postinstall(self):
+        '''Do some post-install setup work with runtime-postinstall.tmpl'''
+        # copy configdir into runtime root beforehand
+        configdir = joinpaths(self._runner.templatedir,"config_files")
+        configdir_path = "tmp/config_files"
+        fullpath = joinpaths(self.vars.root, configdir_path)
+        if os.path.exists(fullpath):
+            remove(fullpath)
+        copytree(configdir, fullpath)
+        self._runner.run("runtime-postinstall.tmpl", configdir=configdir_path)
+
+[docs]    def cleanup(self):
+        '''Remove unneeded packages and files with runtime-cleanup.tmpl'''
+        self._runner.run("runtime-cleanup.tmpl")
+
+[docs]    def writepkgsizes(self, pkgsizefile):
+        '''debugging data: write a big list of pkg sizes'''
+        fobj = open(pkgsizefile, "w")
+        getsize = lambda f: os.lstat(f).st_size if os.path.exists(f) else 0
+        q = self.dbo.sack.query()
+        for p in sorted(q.installed()):
+            pkgsize = sum(getsize(joinpaths(self.vars.root,f)) for f in p.files)
+            fobj.write("{0.name}.{0.arch}: {1}\n".format(p, pkgsize))
+
+[docs]    def generate_module_data(self):
+        root = self.vars.root
+        moddir = joinpaths(root, "lib/modules/")
+        for kver in os.listdir(moddir):
+            ksyms = joinpaths(root, "boot/System.map-%s" % kver)
+            logger.info("doing depmod and module-info for %s", kver)
+            runcmd(["depmod", "-a", "-F", ksyms, "-b", root, kver])
+            generate_module_info(moddir+kver, outfile=moddir+"module-info")
+
+[docs]    def create_runtime(self, outfile="/var/tmp/squashfs.img", compression="xz", compressargs=None, size=2):
+        # make live rootfs image - must be named "LiveOS/rootfs.img" for dracut
+        compressargs = compressargs or []
+        workdir = joinpaths(os.path.dirname(outfile), "runtime-workdir")
+        os.makedirs(joinpaths(workdir, "LiveOS"))
+
+        imgutils.mkrootfsimg(self.vars.root, joinpaths(workdir, "LiveOS/rootfs.img"),
+                             "Anaconda", size=size)
+
+        # squash the live rootfs and clean up workdir
+        imgutils.mksquashfs(workdir, outfile, compression, compressargs)
+        remove(workdir)
+
+[docs]    def finished(self):
+        """ Done using RuntimeBuilder
+
+        Close the dnf base object
+        """
+        self.dbo.close()
+
+[docs]class TreeBuilder(object):
+    '''Builds the arch-specific boot images.
+    inroot should be the installtree root (the newly-built runtime dir)'''
+    def __init__(self, product, arch, inroot, outroot, runtime, isolabel, domacboot=True, doupgrade=True, templatedir=None, add_templates=None, add_template_vars=None, workdir=None):
+
+        # NOTE: if you pass an arg named "runtime" to a mako template it'll
+        # clobber some mako internal variables - hence "runtime_img".
+        self.vars = DataHolder(arch=arch, product=product, runtime_img=runtime,
+                               runtime_base=basename(runtime),
+                               inroot=inroot, outroot=outroot,
+                               basearch=arch.basearch, libdir=arch.libdir,
+                               isolabel=isolabel, udev=udev_escape, domacboot=domacboot, doupgrade=doupgrade,
+                               workdir=workdir, lower=string_lower)
+        self._runner = LoraxTemplateRunner(inroot, outroot, templatedir=templatedir)
+        self._runner.defaults = self.vars
+        self.add_templates = add_templates or []
+        self.add_template_vars = add_template_vars or {}
+        self.templatedir = templatedir
+        self.treeinfo_data = None
+
+    @property
+[docs]    def kernels(self):
+        return findkernels(root=self.vars.inroot)
+
+[docs]    def rebuild_initrds(self, add_args=None, backup="", prefix=""):
+        '''Rebuild all the initrds in the tree. If backup is specified, each
+        initrd will be renamed with backup as a suffix before rebuilding.
+        If backup is empty, the existing initrd files will be overwritten.
+        If suffix is specified, the existing initrd is untouched and a new
+        image is built with the filename "${prefix}-${kernel.version}.img"
+        '''
+        add_args = add_args or []
+        dracut = ["dracut", "--nomdadmconf", "--nolvmconf"] + add_args
+        if not backup:
+            dracut.append("--force")
+
+        kernels = [kernel for kernel in self.kernels if hasattr(kernel, "initrd")]
+        if not kernels:
+            raise Exception("No initrds found, cannot rebuild_initrds")
+
+        # Hush some dracut warnings. TODO: bind-mount proc in place?
+        open(joinpaths(self.vars.inroot,"/proc/modules"),"w")
+        for kernel in kernels:
+            if prefix:
+                idir = os.path.dirname(kernel.initrd.path)
+                outfile = joinpaths(idir, prefix+'-'+kernel.version+'.img')
+            else:
+                outfile = kernel.initrd.path
+            logger.info("rebuilding %s", outfile)
+            if backup:
+                initrd = joinpaths(self.vars.inroot, outfile)
+                os.rename(initrd, initrd + backup)
+            cmd = dracut + [outfile, kernel.version]
+            runcmd(cmd, root=self.vars.inroot)
+
+            # ppc64 cannot boot images > 32MiB, check size and warn
+            if self.vars.arch.basearch in ("ppc64", "ppc64le") and os.path.exists(outfile):
+                st = os.stat(outfile)
+                if st.st_size > 32 * 1024 * 1024:
+                    logging.warning("ppc64 initrd %s is > 32MiB", outfile)
+
+        os.unlink(joinpaths(self.vars.inroot,"/proc/modules"))
+
+[docs]    def build(self):
+        templatefile = templatemap[self.vars.arch.basearch]
+        for tmpl in self.add_templates:
+            self._runner.run(tmpl, **self.add_template_vars)
+        self._runner.run(templatefile, kernels=self.kernels)
+        self.treeinfo_data = self._runner.results.treeinfo
+        self.implantisomd5()
+
+[docs]    def implantisomd5(self):
+        for _section, data in self.treeinfo_data.items():
+            if 'boot.iso' in data:
+                iso = joinpaths(self.vars.outroot, data['boot.iso'])
+                runcmd(["implantisomd5", iso])
+
+    @property
+[docs]    def dracut_hooks_path(self):
+        """ Return the path to the lorax dracut hooks scripts
+
+            Use the configured share dir if it is setup,
+            otherwise default to /usr/share/lorax/dracut_hooks
+        """
+        if self.templatedir:
+            return joinpaths(self.templatedir, "dracut_hooks")
+        else:
+            return "/usr/share/lorax/dracut_hooks"
+
+[docs]    def copy_dracut_hooks(self, hooks):
+        """ Copy the hook scripts in hooks into the installroot's /tmp/
+        and return a list of commands to pass to dracut when creating the
+        initramfs
+
+        hooks is a list of tuples with the name of the hook script and the
+        target dracut hook directory
+        (eg. [("99anaconda-copy-ks.sh", "/lib/dracut/hooks/pre-pivot")])
+        """
+        dracut_commands = []
+        for hook_script, dracut_path in hooks:
+            src = joinpaths(self.dracut_hooks_path, hook_script)
+            if not os.path.exists(src):
+                logger.error("Missing lorax dracut hook script %s", (src))
+                continue
+            dst = joinpaths(self.vars.inroot, "/tmp/", hook_script)
+            copy2(src, dst)
+            dracut_commands += ["--include", joinpaths("/tmp/", hook_script),
+                                dracut_path]
+        return dracut_commands
+
+#### TreeBuilder helper functions
+
+[docs]def findkernels(root="/", kdir="boot"):
+    # To find possible flavors, awk '/BuildKernel/ { print $4 }' kernel.spec
+    flavors = ('debug', 'PAE', 'PAEdebug', 'smp', 'xen', 'lpae')
+    kre = re.compile(r"vmlinuz-(?P<version>.+?\.(?P<arch>[a-z0-9_]+)"
+                     r"(.(?P<flavor>{0}))?)$".format("|".join(flavors)))
+    kernels = []
+    bootfiles = os.listdir(joinpaths(root, kdir))
+    for f in bootfiles:
+        match = kre.match(f)
+        if match:
+            kernel = DataHolder(path=joinpaths(kdir, f))
+            kernel.update(match.groupdict()) # sets version, arch, flavor
+            kernels.append(kernel)
+
+    # look for associated initrd/initramfs/etc.
+    for kernel in kernels:
+        for f in bootfiles:
+            if f.endswith('-'+kernel.version+'.img'):
+                imgtype, _rest = f.split('-',1)
+                # special backwards-compat case
+                if imgtype == 'initramfs':
+                    imgtype = 'initrd'
+                kernel[imgtype] = DataHolder(path=joinpaths(kdir, f))
+
+    logger.debug("kernels=%s", kernels)
+    return kernels
+
+# udev whitelist: 'a-zA-Z0-9#+.:=@_-' (see is_whitelisted in libudev-util.c)
+udev_blacklist=' !"$%&\'()*,/;<>?[\\]^`{|}~' # ASCII printable, minus whitelist
+udev_blacklist += ''.join(chr(i) for i in range(32)) # ASCII non-printable
+[docs]def udev_escape(label):
+    out = ''
+    for ch in label:
+        out += ch if ch not in udev_blacklist else '\\x%02x' % ord(ch)
+    return out
+
+[docs]def string_lower(string):
+    """ Return a lowercase string.
+
+    :param string: String to lowercase
+
+    This is used as a filter in the templates.
+    """
+    return string.lower()
+

+ +

+ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/_modules/pylorax/treeinfo.html b/_modules/pylorax/treeinfo.html new file mode 100644 index 00000000..8ef20486 --- /dev/null +++ b/_modules/pylorax/treeinfo.html @@ -0,0 +1,245 @@ + + + + + + + + + + pylorax.treeinfo — Lorax 23.8 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+ + + + +

+ + + + + + +

+ +

Source code for pylorax.treeinfo

+#
+# treeinfo.py
+#
+# Copyright (C) 2010-2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Red Hat Author(s):  Martin Gracik <mgracik@redhat.com>
+#
+
+import logging
+logger = logging.getLogger("pylorax.treeinfo")
+
+import configparser
+import time
+
+
+[docs]class TreeInfo(object):
+
+    def __init__(self, product, version, variant, basearch,
+                 packagedir=""):
+
+        self.c = configparser.ConfigParser()
+
+        section = "general"
+        data = {"timestamp": str(time.time()),
+                "family": product,
+                "version": version,
+                "name": "%s-%s" % (product, version),
+                "variant": variant or "",
+                "arch": basearch,
+                "packagedir": packagedir}
+
+        self.c.add_section(section)
+        list(self.c.set(section, key, value) for key, value in data.items())
+
+[docs]    def add_section(self, section, data):
+        if not self.c.has_section(section):
+            self.c.add_section(section)
+
+        list(self.c.set(section, key, value) for key, value in data.items())
+
+[docs]    def write(self, outfile):
+        logger.info("writing .treeinfo file")
+        with open(outfile, "w") as fobj:
+            self.c.write(fobj)
+

+ +

+ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/_sources/index.txt b/_sources/index.txt new file mode 100644 index 00000000..c9b962d8 --- /dev/null +++ b/_sources/index.txt @@ -0,0 +1,27 @@ +.. Lorax documentation master file, created by + sphinx-quickstart on Wed Apr 8 13:46:00 2015. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to Lorax's documentation! +================================= + +Contents: + +.. toctree:: + :maxdepth: 1 + + intro + lorax + livemedia-creator + product-images + modules + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/_sources/intro.txt b/_sources/intro.txt new file mode 100644 index 00000000..01857ee9 --- /dev/null +++ b/_sources/intro.txt @@ -0,0 +1,67 @@ +Introduction to Lorax +===================== + +I am the Lorax. I speak for the trees [and images]. + +Lorax is used to build the Anaconda Installer boot.iso, it consists of a +library, pylorax, a set of templates, and the lorax script. Its operation +is driven by a customized set of Mako templates that lists the packages +to be installed, steps to execute to remove unneeded files, and creation +of the iso for all of the supported architectures. + + + + + + +Before Lorax +============ + +Tree building tools such as pungi and revisor rely on 'buildinstall' in +anaconda/scripts/ to produce the boot images and other such control files +in the final tree. The existing buildinstall scripts written in a mix of +bash and Python are unmaintainable. Lorax is an attempt to replace them +with something more flexible. + + +EXISTING WORKFLOW: + +pungi and other tools call scripts/buildinstall, which in turn call other +scripts to do the image building and data generation. Here's how it +currently looks: + + -> buildinstall + * process command line options + * write temporary yum.conf to point to correct repo + * find anaconda release RPM + * unpack RPM, pull in those versions of upd-instroot, mk-images, + maketreeinfo.py, makestamp.py, and buildinstall + + -> call upd-instroot + + -> call maketreeinfo.py + + -> call mk-images (which figures out which mk-images.ARCH to call) + + -> call makestamp.py + + * clean up + + +PROBLEMS: + +The existing workflow presents some problems with maintaining the scripts. +First, almost all knowledge of what goes in to the stage 1 and stage 2 +images lives in upd-instroot. The mk-images* scripts copy things from the +root created by upd-instroot in order to build the stage 1 image, though +it's not completely clear from reading the scripts. + + +NEW IDEAS: + +Create a new central driver with all information living in Python modules. +Configuration files will provide the knowledge previously contained in the +upd-instroot and mk-images* scripts. + + + diff --git a/_sources/livemedia-creator.txt b/_sources/livemedia-creator.txt new file mode 100644 index 00000000..3b8c3a2b --- /dev/null +++ b/_sources/livemedia-creator.txt @@ -0,0 +1,500 @@ +livemedia-creator +================= + +:Authors: + Brian C. Lane + +livemedia-creator uses `Anaconda `_, +`kickstart `_ and `Lorax +`_ to create bootable media that use the +same install path as a normal system installation. It can be used to make live +isos, bootable (partitioned) disk images, tarfiles, and filesystem images for +use with virtualization and container solutions like libvirt, docker, and +OpenStack. + +The general idea is to use virt-install with kickstart and an Anaconda boot.iso +to install into a disk image and then use the disk image to create the bootable +media. + +livemedia-creator --help will describe all of the options available. At the +minimum you need: + +``--make-iso`` to create a final bootable .iso or one of the other ``--make-*`` options. + +``--iso`` to specify the Anaconda install media to use with virt-install + +``--ks`` to select the kickstart file describing what to install. + +To use livemedia-creator with virt-install you will need to install the +following packages, as well as have libvirtd setup correctly. + +* ``virt-install`` +* ``libvirt-python`` + +If you are going to be using Anaconda directly, with ``--no-virt`` mode, make sure +you have the anaconda package installed. You can use the anaconda-tui package +to save a bit of space on the build system. + +Conventions used in this document: + +``lmc`` is an abbreviation for livemedia-creator. + +``builder`` is the system where livemedia-creator is being run + +``image`` is the disk image being created by running livemedia-creator + + +Quickstart +---------- + +Run this to create a bootable live iso:: + + sudo livemedia-creator --make-iso \ + --iso=/extra/iso/boot.iso --ks=./docs/fedora-livemedia.ks + +You can run it directly from the lorax git repo like this:: + + sudo PATH=./src/sbin/:$PATH PYTHONPATH=./src/ ./src/sbin/livemedia-creator \ + --make-iso --iso=/extra/iso/boot.iso \ + --ks=./docs/fedora-livemedia.ks --lorax-templates=./share/ + +If you want to watch the install you can pass ``--vnc vnc`` and use a vnc client +to connect to localhost:0 + +This is usually a good idea when testing changes to the kickstart. lmc tries +to monitor the logs for fatal errors, but may not catch everything. + + +How ISO creation works +---------------------- + +There are 2 stages, the install stage which produces a disk or filesystem image +as its output, and the boot media creation which uses the image as its input. +Normally you would run both stages, but it is possible to stop after the +install stage, by using ``--image-only``, or to skip the install stage and use +a previously created disk image by passing ``--disk-image`` or ``--fs-image`` + +When creating an iso virt-install boots using the passed Anaconda installer iso +and installs the system based on the kickstart. The ``%post`` section of the +kickstart is used to customize the installed system in the same way that +current spin-kickstarts do. + +livemedia-creator monitors the install process for problems by watching the +install logs. They are written to the current directory or to the base +directory specified by the --logfile command. You can also monitor the install +by passing ``--vnc vnc`` and using a vnc client. This is recommended when first +modifying a kickstart, since there are still places where Anaconda may get +stuck without the log monitor catching it. + +The output from this process is a partitioned disk image. kpartx can be used +to mount and examine it when there is a problem with the install. It can also +be booted using kvm. + +When creating an iso the disk image's / partition is copied into a formatted +disk image which is then used as the input to lorax for creation of the final +media. + +The final image is created by lorax, using the templates in /usr/share/lorax/ +or the directory specified by ``--lorax-templates`` + +Currently the standard lorax templates are used to make a bootable iso, but +it should be possible to modify them to output other results. They are +written using the Mako template system which is very flexible. + + +Kickstarts +---------- + +The docs/ directory includes several example kickstarts, one to create a live +desktop iso using GNOME, and another to create a minimal disk image. When +creating your own kickstarts you should start with the minimal example, it +includes several needed packages that are not always included by dependencies. + +Or you can use existing spin kickstarts to create live media with a few +changes. Here are the steps I used to convert the Fedora XFCE spin. + +1. Flatten the xfce kickstart using ksflatten +2. Add zerombr so you don't get the disk init dialog +3. Add clearpart --all +4. Add swap partition +5. bootloader target +6. Add shutdown to the kickstart +7. Add network --bootproto=dhcp --activate to activate the network + This works for F16 builds but for F15 and before you need to pass + something on the cmdline that activate the network, like sshd: + + ``livemedia-creator --kernel-args="sshd"`` + +8. Add a root password:: + + rootpw rootme + network --bootproto=dhcp --activate + zerombr + clearpart --all + bootloader --location=mbr + part swap --size=512 + shutdown + +9. In the livesys script section of the %post remove the root password. This + really depends on how the spin wants to work. You could add the live user + that you create to the %wheel group so that sudo works if you wanted to. + + ``passwd -d root > /dev/null`` + +10. Remove /etc/fstab in %post, dracut handles mounting the rootfs + + ``cat /dev/null > /dev/fstab`` + + Do this only for live iso's, the filesystem will be mounted read only if + there is no /etc/fstab + +11. Don't delete initramfs files from /boot in %post +12. Have dracut-config-generic, grub-efi, memtest86+ and syslinux in the package + list. +13. Omit dracut-config-rescue from the %package list: ``-dracut-config-rescue`` + +One drawback to using virt-install is that it pulls the packages from +the repo each time you run it. To speed things up you either need a local +mirror of the packages, or you can use a caching proxy. When using a proxy +you pass it to livemedia-creator like this: + + ``--proxy=http://proxy.yourdomain.com:3128`` + +You also need to use a specific mirror instead of mirrormanager so that the +packages will get cached, so your kickstart url would look like: + + ``url --url="http://dl.fedoraproject.org/pub/fedora/linux/development/17/x86_64/os/"`` + +You can also add an update repo, but don't name it updates. Add --proxy to +it as well. + + +Anaconda image install (no-virt) +-------------------------------- + +You can create images without using virt-install by passing ``--no-virt`` on the +cmdline. This will use Anaconda's directory install feature to handle the install. +There are a couple of things to keep in mind when doing this: + +1. It will be most reliable when building images for the same release that the + host is running. Because Anaconda has expectations about the system it is + running under you may encounter strange bugs if you try to build newer or + older releases. + +2. Make sure selinux is set to permissive or disabled. It won't install + correctly with selinux set to enforcing yet. + +3. It may totally trash your host. So far I haven't had this happen, but the + possibility exists that a bug in Anaconda could result in it operating on + real devices. I recommend running it in a virt or on a system that you can + afford to lose all data from. + +The logs from anaconda will be placed in an ./anaconda/ directory in either +the current directory or in the directory used for --logfile + +Example cmdline: + +``sudo livemedia-creator --make-iso --no-virt --ks=./fedora-livemedia.ks`` + + +AMI Images +---------- + +Amazon EC2 images can be created by using the --make-ami switch and an appropriate +kickstart file. All of the work to customize the image is handled by the kickstart. +The example currently included was modified from the cloud-kickstarts version so +that it would work with livemedia-creator. + +Example cmdline: + +``sudo livemedia-creator --make-ami --iso=/path/to/boot.iso --ks=./docs/fedora-livemedia-ec2.ks`` + +This will produce an ami-root.img file in the working directory. + +At this time I have not tested the image with EC2. Feedback would be welcome. + + +Appliance Creation +------------------ + +livemedia-creator can now replace appliance-tools by using the --make-appliance +switch. This will create the partitioned disk image and an XML file that can be +used with virt-image to setup a virtual system. + +The XML is generated using the Mako template from +/usr/share/lorax/appliance/libvirt.xml You can use a different template by +passing ``--app-template


	+ p
	+ `pylorax`	+
	+ `pylorax.base`	+
	+ `pylorax.buildstamp`	+
	+ `pylorax.decorators`	+
	+ `pylorax.discinfo`	+
	+ `pylorax.dnfhelper`	+
	+ `pylorax.executils`	+
	+ `pylorax.imgutils`	+
	+ `pylorax.ltmpl`	+
	+ `pylorax.output`	+
	+ `pylorax.sysutils`	+
	+ `pylorax.treebuilder`	+
	+ `pylorax.treeinfo`	+

All modules for which code is available

Source code for pylorax

Source code for pylorax.base

Source code for pylorax.buildstamp

Source code for pylorax.decorators

Source code for pylorax.discinfo

Source code for pylorax.dnfhelper

Source code for pylorax.executils

Source code for pylorax.imgutils

Source code for pylorax.ltmpl

Source code for pylorax.sysutils

Source code for pylorax.treebuilder

Source code for pylorax.treeinfo

' + _('Searching') + '

Index

A

B

C

D

E

F

G

H

I

J

K

L

M

P

R

S

T

U

W

Welcome to Lorax’s documentation!¶

Indices and tables¶

Introduction to Lorax¶

Before Lorax¶

livemedia-creator¶

Quickstart¶

How ISO creation works¶

Kickstarts¶

Anaconda image install (no-virt)¶

AMI Images¶

Appliance Creation¶

Filesystem Image Creation¶

TAR File Creation¶

Live Image for PXE Boot¶

Atomic Live Image for PXE Boot¶

Using Mock to Create Images¶

OpenStack Image Creation¶

Docker Image Creation¶

Debugging problems¶

Hacking¶

lorax¶

pylorax¶

Product and Updates Images¶

Python Module Index

pylorax package¶

Submodules¶

pylorax.base module¶

pylorax.buildstamp module¶

pylorax.decorators module¶

pylorax.discinfo module¶

pylorax.dnfhelper module¶

pylorax.executils module¶

pylorax.imgutils module¶

pylorax.ltmpl module¶

pylorax.output module¶

pylorax.sysutils module¶

pylorax.treebuilder module¶

pylorax.treeinfo module¶

Module contents¶

Welcome to lorax’s documentation!¶

Indices and tables¶