diff --git a/docs/html/.buildinfo b/docs/html/.buildinfo new file mode 100644 index 00000000..53c96413 --- /dev/null +++ b/docs/html/.buildinfo @@ -0,0 +1,4 @@ +# Sphinx build info version 1 +# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. +config: d776326856759e24083884fa308a1bc8 +tags: 645f666f9bcd5a90fca523b33c5a78b7 diff --git a/docs/html/.doctrees/environment.pickle b/docs/html/.doctrees/environment.pickle new file mode 100644 index 00000000..e14fcef7 Binary files /dev/null and b/docs/html/.doctrees/environment.pickle differ diff --git a/docs/html/.doctrees/index.doctree b/docs/html/.doctrees/index.doctree new file mode 100644 index 00000000..f8f59a4d Binary files /dev/null and b/docs/html/.doctrees/index.doctree differ diff --git a/docs/html/.doctrees/intro.doctree b/docs/html/.doctrees/intro.doctree new file mode 100644 index 00000000..8f6dd3a4 Binary files /dev/null and b/docs/html/.doctrees/intro.doctree differ diff --git a/docs/html/.doctrees/livemedia-creator.doctree b/docs/html/.doctrees/livemedia-creator.doctree new file mode 100644 index 00000000..5ff4e3db Binary files /dev/null and b/docs/html/.doctrees/livemedia-creator.doctree differ diff --git a/docs/html/.doctrees/lorax.doctree b/docs/html/.doctrees/lorax.doctree new file mode 100644 index 00000000..74d4ee94 Binary files /dev/null and b/docs/html/.doctrees/lorax.doctree differ diff --git a/docs/html/.doctrees/modules.doctree b/docs/html/.doctrees/modules.doctree new file mode 100644 index 00000000..094fe54a Binary files /dev/null and b/docs/html/.doctrees/modules.doctree differ diff --git a/docs/html/.doctrees/product-images.doctree b/docs/html/.doctrees/product-images.doctree new file mode 100644 index 00000000..d3dd62e4 Binary files /dev/null and b/docs/html/.doctrees/product-images.doctree differ diff --git a/docs/html/.doctrees/pylorax.doctree b/docs/html/.doctrees/pylorax.doctree new file mode 100644 index 00000000..707dd925 Binary files /dev/null and b/docs/html/.doctrees/pylorax.doctree differ diff --git a/docs/html/.doctrees/source/index.doctree b/docs/html/.doctrees/source/index.doctree new file mode 100644 index 00000000..a1aebf96 Binary files /dev/null and b/docs/html/.doctrees/source/index.doctree differ diff --git a/docs/html/_modules/index.html b/docs/html/_modules/index.html new file mode 100644 index 00000000..894ad67e --- /dev/null +++ b/docs/html/_modules/index.html @@ -0,0 +1,195 @@ + + + + + + + + + + Overview: module code — Lorax 23.8 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + +
+ + + + + + +
+
+
+
    +
  • Docs »
    • + +
  • Overview: module code
    • +
  • + +
    • +
+
+
+
+ +

All modules for which code is available

+ + +
+ +
+
+ +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/html/_modules/pylorax.html b/docs/html/_modules/pylorax.html new file mode 100644 index 00000000..d96055bb --- /dev/null +++ b/docs/html/_modules/pylorax.html @@ -0,0 +1,561 @@ + + + + + + + + + + pylorax — Lorax 23.8 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + +
+ + + + + + +
+
+
+ +
+
+
+ +

Source code for pylorax

+#
+# __init__.py
+#
+# Copyright (C) 2010-2015  Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Red Hat Author(s):  Martin Gracik <mgracik@redhat.com>
+#                     David Cantrell <dcantrell@redhat.com>
+#                     Will Woods <wwoods@redhat.com>
+
+# set up logging
+import logging
+logger = logging.getLogger("pylorax")
+logger.addHandler(logging.NullHandler())
+
+import sys
+import os
+import configparser
+import tempfile
+import locale
+from subprocess import CalledProcessError
+import selinux
+
+from pylorax.base import BaseLoraxClass, DataHolder
+import pylorax.output as output
+
+import dnf
+
+from pylorax.sysutils import joinpaths, remove, linktree
+
+from pylorax.treebuilder import RuntimeBuilder, TreeBuilder
+from pylorax.buildstamp import BuildStamp
+from pylorax.treeinfo import TreeInfo
+from pylorax.discinfo import DiscInfo
+from pylorax.executils import runcmd, runcmd_output
+
+# List of drivers to remove on ppc64 arch to keep initrd < 32MiB
+REMOVE_PPC64_DRIVERS = "floppy scsi_debug nouveau radeon cirrus mgag200"
+REMOVE_PPC64_MODULES = "drm plymouth"
+
+
[docs]class ArchData(DataHolder):
+    lib64_arches = ("x86_64", "ppc64", "ppc64le", "s390x", "ia64", "aarch64")
+    bcj_arch = dict(i386="x86", x86_64="x86",
+                    ppc="powerpc", ppc64="powerpc", ppc64le="powerpc",
+                    arm="arm", armhfp="arm")
+
+    def __init__(self, buildarch):
+        super(ArchData, self).__init__()
+        self.buildarch = buildarch
+        self.basearch = dnf.arch.basearch(buildarch)
+        self.libdir = "lib64" if self.basearch in self.lib64_arches else "lib"
+        self.bcj = self.bcj_arch.get(self.basearch)
+

+
[docs]class Lorax(BaseLoraxClass):
+
+    def __init__(self):
+        BaseLoraxClass.__init__(self)
+        self._configured = False
+        self.product = None
+        self.workdir = None
+        self.arch = None
+        self.conf = None
+        self.inroot = None
+        self.debug = False
+        self.outputdir = None
+
+        # set locale to C
+        locale.setlocale(locale.LC_ALL, 'C')
+
+
[docs]    def configure(self, conf_file="/etc/lorax/lorax.conf"):
+        self.conf = configparser.SafeConfigParser()
+
+        # set defaults
+        self.conf.add_section("lorax")
+        self.conf.set("lorax", "debug", "1")
+        self.conf.set("lorax", "sharedir", "/usr/share/lorax")
+        self.conf.set("lorax", "logdir", "/var/log/lorax")
+
+        self.conf.add_section("output")
+        self.conf.set("output", "colors", "1")
+        self.conf.set("output", "encoding", "utf-8")
+        self.conf.set("output", "ignorelist", "/usr/share/lorax/ignorelist")
+
+        self.conf.add_section("templates")
+        self.conf.set("templates", "ramdisk", "ramdisk.ltmpl")
+
+        self.conf.add_section("compression")
+        self.conf.set("compression", "type", "xz")
+        self.conf.set("compression", "args", "")
+        self.conf.set("compression", "bcj", "on")
+
+        # read the config file
+        if os.path.isfile(conf_file):
+            self.conf.read(conf_file)
+
+        # set up the output
+        self.debug = self.conf.getboolean("lorax", "debug")
+        output_level = output.DEBUG if self.debug else output.INFO
+
+        if sys.stdout.isatty():
+            colors = self.conf.getboolean("output", "colors")
+        else:
+            colors = False
+        encoding = self.conf.get("output", "encoding")
+
+        self.output.basic_config(output_level=output_level,
+                                 colors=colors, encoding=encoding)
+
+        ignorelist = self.conf.get("output", "ignorelist")
+        if os.path.isfile(ignorelist):
+            with open(ignorelist, "r") as fobj:
+                for line in fobj:
+                    line = line.strip()
+                    if line and not line.startswith("#"):
+                        self.output.ignore(line)
+
+        # cron does not have sbin in PATH,
+        # so we have to add it ourselves
+        os.environ["PATH"] = "{0}:/sbin:/usr/sbin".format(os.environ["PATH"])
+
+        # remove some environmental variables that can cause problems with package scripts
+        env_remove = ('DISPLAY', 'DBUS_SESSION_BUS_ADDRESS')
+        list(os.environ.pop(k) for k in env_remove if k in os.environ)
+
+        self._configured = True
+

+
[docs]    def init_stream_logging(self):
+        sh = logging.StreamHandler()
+        sh.setLevel(logging.INFO)
+        logger.addHandler(sh)
+

+
[docs]    def init_file_logging(self, logdir, logname="pylorax.log"):
+        fh = logging.FileHandler(filename=joinpaths(logdir, logname), mode="w")
+        fh.setLevel(logging.DEBUG)
+        logger.addHandler(fh)
+

+
[docs]    def run(self, dbo, product, version, release, variant="", bugurl="",
+            isfinal=False, workdir=None, outputdir=None, buildarch=None, volid=None,
+            domacboot=True, doupgrade=True, remove_temp=False,
+            installpkgs=None,
+            size=2,
+            add_templates=None,
+            add_template_vars=None,
+            add_arch_templates=None,
+            add_arch_template_vars=None):
+
+        assert self._configured
+
+        installpkgs = installpkgs or []
+
+        # get lorax version
+        try:
+            import pylorax.version
+        except ImportError:
+            vernum = "devel"
+        else:
+            vernum = pylorax.version.num
+
+        if domacboot:
+            try:
+                runcmd(["rpm", "-q", "hfsplus-tools"])
+            except CalledProcessError:
+                logger.critical("you need to install hfsplus-tools to create mac images")
+                sys.exit(1)
+
+        # set up work directory
+        self.workdir = workdir or tempfile.mkdtemp(prefix="pylorax.work.")
+        if not os.path.isdir(self.workdir):
+            os.makedirs(self.workdir)
+
+        # set up log directory
+        logdir = self.conf.get("lorax", "logdir")
+        if not os.path.isdir(logdir):
+            os.makedirs(logdir)
+
+        self.init_stream_logging()
+        self.init_file_logging(logdir)
+
+        logger.debug("version is %s", vernum)
+        logger.debug("using work directory %s", self.workdir)
+        logger.debug("using log directory %s", logdir)
+
+        # set up output directory
+        self.outputdir = outputdir or tempfile.mkdtemp(prefix="pylorax.out.")
+        if not os.path.isdir(self.outputdir):
+            os.makedirs(self.outputdir)
+        logger.debug("using output directory %s", self.outputdir)
+
+        # do we have root privileges?
+        logger.info("checking for root privileges")
+        if not os.geteuid() == 0:
+            logger.critical("no root privileges")
+            sys.exit(1)
+
+        # is selinux disabled?
+        # With selinux in enforcing mode the rpcbind package required for
+        # dracut nfs module, which is in turn required by anaconda module,
+        # will not get installed, because it's preinstall scriptlet fails,
+        # resulting in an incomplete initial ramdisk image.
+        # The reason is that the scriptlet runs tools from the shadow-utils
+        # package in chroot, particularly groupadd and useradd to add the
+        # required rpc group and rpc user. This operation fails, because
+        # the selinux context on files in the chroot, that the shadow-utils
+        # tools need to access (/etc/group, /etc/passwd, /etc/shadow etc.),
+        # is wrong and selinux therefore disallows access to these files.
+        logger.info("checking the selinux mode")
+        if selinux.is_selinux_enabled() and selinux.security_getenforce():
+            logger.critical("selinux must be disabled or in Permissive mode")
+            sys.exit(1)
+
+        # do we have a proper dnf base object?
+        logger.info("checking dnf base object")
+        if not isinstance(dbo, dnf.Base):
+            logger.critical("no dnf base object")
+            sys.exit(1)
+        self.inroot = dbo.conf.installroot
+        logger.debug("using install root: %s", self.inroot)
+
+        if not buildarch:
+            buildarch = get_buildarch(dbo)
+
+        logger.info("setting up build architecture")
+        self.arch = ArchData(buildarch)
+        for attr in ('buildarch', 'basearch', 'libdir'):
+            logger.debug("self.arch.%s = %s", attr, getattr(self.arch,attr))
+
+        logger.info("setting up build parameters")
+        product = DataHolder(name=product, version=version, release=release,
+                             variant=variant, bugurl=bugurl, isfinal=isfinal)
+        self.product = product
+        logger.debug("product data: %s", product)
+
+        # NOTE: if you change isolabel, you need to change pungi to match, or
+        # the pungi images won't boot.
+        isolabel = volid or "%s-%s-%s" % (product, version, self.arch.basearch)
+
+        if len(isolabel) > 32:
+            logger.fatal("the volume id cannot be longer than 32 characters")
+            sys.exit(1)
+
+        templatedir = self.conf.get("lorax", "sharedir")
+        # NOTE: rb.root = dbo.conf.installroot (== self.inroot)
+        rb = RuntimeBuilder(product=self.product, arch=self.arch,
+                            dbo=dbo, templatedir=templatedir,
+                            installpkgs=installpkgs,
+                            add_templates=add_templates,
+                            add_template_vars=add_template_vars)
+
+        logger.info("installing runtime packages")
+        rb.install()
+
+        # write .buildstamp
+        buildstamp = BuildStamp(self.product.name, self.product.version,
+                                self.product.bugurl, self.product.isfinal, self.arch.buildarch)
+
+        buildstamp.write(joinpaths(self.inroot, ".buildstamp"))
+
+        if self.debug:
+            rb.writepkglists(joinpaths(logdir, "pkglists"))
+            rb.writepkgsizes(joinpaths(logdir, "original-pkgsizes.txt"))
+
+        logger.info("doing post-install configuration")
+        rb.postinstall()
+
+        # write .discinfo
+        discinfo = DiscInfo(self.product.release, self.arch.basearch)
+        discinfo.write(joinpaths(self.outputdir, ".discinfo"))
+
+        logger.info("backing up installroot")
+        installroot = joinpaths(self.workdir, "installroot")
+        linktree(self.inroot, installroot)
+
+        logger.info("generating kernel module metadata")
+        rb.generate_module_data()
+
+        logger.info("cleaning unneeded files")
+        rb.cleanup()
+
+        if self.debug:
+            rb.writepkgsizes(joinpaths(logdir, "final-pkgsizes.txt"))
+
+        logger.info("creating the runtime image")
+        runtime = "images/install.img"
+        compression = self.conf.get("compression", "type")
+        compressargs = self.conf.get("compression", "args").split()     # pylint: disable=no-member
+        if self.conf.getboolean("compression", "bcj"):
+            if self.arch.bcj:
+                compressargs += ["-Xbcj", self.arch.bcj]
+            else:
+                logger.info("no BCJ filter for arch %s", self.arch.basearch)
+        rb.create_runtime(joinpaths(installroot,runtime),
+                          compression=compression, compressargs=compressargs,
+                          size=size)
+        rb.finished()
+
+        logger.info("preparing to build output tree and boot images")
+        treebuilder = TreeBuilder(product=self.product, arch=self.arch,
+                                  inroot=installroot, outroot=self.outputdir,
+                                  runtime=runtime, isolabel=isolabel,
+                                  domacboot=domacboot, doupgrade=doupgrade,
+                                  templatedir=templatedir,
+                                  add_templates=add_arch_templates,
+                                  add_template_vars=add_arch_template_vars,
+                                  workdir=self.workdir)
+
+        logger.info("rebuilding initramfs images")
+        dracut_args = ["--xz", "--install", "/.buildstamp"]
+        anaconda_args = dracut_args + ["--add", "anaconda pollcdrom"]
+
+        # ppc64 cannot boot an initrd > 32MiB so remove some drivers
+        if self.arch.basearch in ("ppc64", "ppc64le"):
+            dracut_args.extend(["--omit-drivers", REMOVE_PPC64_DRIVERS])
+
+            # Only omit dracut modules from the initrd so that they're kept for
+            # upgrade.img
+            anaconda_args.extend(["--omit", REMOVE_PPC64_MODULES])
+
+        treebuilder.rebuild_initrds(add_args=anaconda_args)
+
+        if doupgrade:
+            # Build upgrade.img. It'd be nice if these could coexist in the same
+            # image, but that would increase the size of the anaconda initramfs,
+            # which worries some people (esp. PPC tftpboot). So they're separate.
+            try:
+                # If possible, use the 'fedup' plymouth theme
+                themes = runcmd_output(['plymouth-set-default-theme', '--list'],
+                                       root=installroot)
+                if 'fedup' in themes.splitlines():
+                    os.environ['PLYMOUTH_THEME_NAME'] = 'fedup'
+            except RuntimeError:
+                pass
+            upgrade_args = dracut_args + ["--add", "system-upgrade"]
+            treebuilder.rebuild_initrds(add_args=upgrade_args, prefix="upgrade")
+
+        logger.info("populating output tree and building boot images")
+        treebuilder.build()
+
+        # write .treeinfo file and we're done
+        treeinfo = TreeInfo(self.product.name, self.product.version,
+                            self.product.variant, self.arch.basearch)
+        for section, data in treebuilder.treeinfo_data.items():
+            treeinfo.add_section(section, data)
+        treeinfo.write(joinpaths(self.outputdir, ".treeinfo"))
+
+        # cleanup
+        if remove_temp:
+            remove(self.workdir)
+
+

+
[docs]def get_buildarch(dbo):
+    # get architecture of the available anaconda package
+    buildarch = None
+    q = dbo.sack.query()
+    a = q.available()
+    for anaconda in a.filter(name="anaconda"):
+        if anaconda.arch != "src":
+            buildarch = anaconda.arch
+            break
+    if not buildarch:
+        logger.critical("no anaconda package in the repository")
+        sys.exit(1)
+
+    return buildarch

+
+ +
+ +
+
+ +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/html/_modules/pylorax/base.html b/docs/html/_modules/pylorax/base.html new file mode 100644 index 00000000..f815c545 --- /dev/null +++ b/docs/html/_modules/pylorax/base.html @@ -0,0 +1,252 @@ + + + + + + + + + + pylorax.base — Lorax 23.8 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + +
+ + + + + + +
+
+
+ +
+
+
+ +

Source code for pylorax.base

+#
+# base.py
+#
+# Copyright (C) 2009-2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Red Hat Author(s):  Martin Gracik <mgracik@redhat.com>
+#
+
+from abc import ABCMeta, abstractmethod
+import sys
+
+import pylorax.output as output
+
+
+
[docs]class BaseLoraxClass(object, metaclass=ABCMeta):
+    @abstractmethod
+    def __init__(self):
+        self.output = output.LoraxOutput()
+
+
[docs]    def pcritical(self, msg, fobj=sys.stdout):
+        self.output.critical(msg, fobj)
+

+
[docs]    def perror(self, msg, fobj=sys.stdout):
+        self.output.error(msg, fobj)
+

+
[docs]    def pwarning(self, msg, fobj=sys.stdout):
+        self.output.warning(msg, fobj)
+

+
[docs]    def pinfo(self, msg, fobj=sys.stdout):
+        self.output.info(msg, fobj)
+

+
[docs]    def pdebug(self, msg, fobj=sys.stdout):
+        self.output.debug(msg, fobj)
+
+

+
[docs]class DataHolder(dict):
+
+    def __init__(self, **kwargs):
+        dict.__init__(self)
+
+        for attr, value in kwargs.items():
+            self[attr] = value
+
+    def __getattr__(self, attr):
+        return self[attr]
+
+    def __setattr__(self, attr, value):
+        self[attr] = value
+
+
[docs]    def copy(self):
+        return DataHolder(**dict.copy(self))

+
+ +
+ +
+
+ +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/html/_modules/pylorax/buildstamp.html b/docs/html/_modules/pylorax/buildstamp.html new file mode 100644 index 00000000..537734e3 --- /dev/null +++ b/docs/html/_modules/pylorax/buildstamp.html @@ -0,0 +1,246 @@ + + + + + + + + + + pylorax.buildstamp — Lorax 23.8 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + +
+ + + + + + +
+
+
+ +
+
+
+ +

Source code for pylorax.buildstamp

+#
+# buildstamp.py
+#
+# Copyright (C) 2010-2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Red Hat Author(s):  Martin Gracik <mgracik@redhat.com>
+#
+
+import logging
+logger = logging.getLogger("pylorax.buildstamp")
+
+import datetime
+
+
+
[docs]class BuildStamp(object):
+
+    def __init__(self, product, version, bugurl, isfinal, buildarch):
+        self.product = product
+        self.version = version
+        self.bugurl = bugurl
+        self.isfinal = isfinal
+
+        now = datetime.datetime.now()
+        now = now.strftime("%Y%m%d%H%M")
+        self.uuid = "{0}.{1}".format(now, buildarch)
+
+
[docs]    def write(self, outfile):
+        # get lorax version
+        try:
+            import pylorax.version
+        except ImportError:
+            vernum = "devel"
+        else:
+            vernum = pylorax.version.num
+
+        logger.info("writing .buildstamp file")
+        with open(outfile, "w") as fobj:
+            fobj.write("[Main]\n")
+            fobj.write("Product={0.product}\n".format(self))
+            fobj.write("Version={0.version}\n".format(self))
+            fobj.write("BugURL={0.bugurl}\n".format(self))
+            fobj.write("IsFinal={0.isfinal}\n".format(self))
+            fobj.write("UUID={0.uuid}\n".format(self))
+            fobj.write("[Compose]\n")
+            fobj.write("Lorax={0}\n".format(vernum))

+
+ +
+ +
+
+ +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/html/_modules/pylorax/decorators.html b/docs/html/_modules/pylorax/decorators.html new file mode 100644 index 00000000..34f44457 --- /dev/null +++ b/docs/html/_modules/pylorax/decorators.html @@ -0,0 +1,218 @@ + + + + + + + + + + pylorax.decorators — Lorax 23.8 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + +
+ + + + + + +
+
+
+ +
+
+
+ +

Source code for pylorax.decorators

+#
+# decorators.py
+#
+# Copyright (C) 2009-2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Red Hat Author(s):  Martin Gracik <mgracik@redhat.com>
+#
+
+
[docs]def singleton(cls):
+    instances = {}
+
+    def get_instance():
+        if cls not in instances:
+            instances[cls] = cls()
+        return instances[cls]
+
+    return get_instance

+
+ +
+ +
+
+ +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/html/_modules/pylorax/discinfo.html b/docs/html/_modules/pylorax/discinfo.html new file mode 100644 index 00000000..651b6e50 --- /dev/null +++ b/docs/html/_modules/pylorax/discinfo.html @@ -0,0 +1,227 @@ + + + + + + + + + + pylorax.discinfo — Lorax 23.8 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + +
+ + + + + + +
+
+
+ +
+
+
+ +

Source code for pylorax.discinfo

+#
+# discinfo.py
+#
+# Copyright (C) 2010-2015  Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Red Hat Author(s):  Martin Gracik <mgracik@redhat.com>
+#
+
+import logging
+logger = logging.getLogger("pylorax.discinfo")
+
+import time
+
+
+
[docs]class DiscInfo(object):
+
+    def __init__(self, release, basearch):
+        self.release = release
+        self.basearch = basearch
+
+
[docs]    def write(self, outfile):
+        logger.info("writing .discinfo file")
+        with open(outfile, "w") as fobj:
+            fobj.write("{0:f}\n".format(time.time()))
+            fobj.write("{0.release}\n".format(self))
+            fobj.write("{0.basearch}\n".format(self))

+
+ +
+ +
+
+ +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/html/_modules/pylorax/dnfhelper.html b/docs/html/_modules/pylorax/dnfhelper.html new file mode 100644 index 00000000..4862ac89 --- /dev/null +++ b/docs/html/_modules/pylorax/dnfhelper.html @@ -0,0 +1,295 @@ + + + + + + + + + + pylorax.dnfhelper — Lorax 23.8 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + +
+ + + + + + +
+
+
+ +
+
+
+ +

Source code for pylorax.dnfhelper

+#
+# dnfhelper.py
+#
+# Copyright (C) 2010-2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Red Hat Author(s):  Martin Gracik <mgracik@redhat.com>
+#                     Brian C. Lane <bcl@redhat.com>
+#
+
+import logging
+logger = logging.getLogger("pylorax.dnfhelper")
+import dnf
+import collections
+import time
+import pylorax.output as output
+
+__all__ = ['LoraxDownloadCallback', 'LoraxRpmCallback']
+
+def _paced(fn):
+    """Execute `fn` no more often then every 2 seconds."""
+    def paced_fn(self, *args):
+        now = time.time()
+        if now - self.last_time < 2:
+            return
+        self.last_time = now
+        return fn(self, *args)
+    return paced_fn
+
+
+
[docs]class LoraxDownloadCallback(dnf.callback.DownloadProgress):
+    def __init__(self):
+        self.downloads = collections.defaultdict(int)
+        self.last_time = time.time()
+        self.total_files = 0
+        self.total_size = 0
+
+        self.pkgno = 0
+        self.total = 0
+
+        self.output = output.LoraxOutput()
+
+    @_paced
+    def _update(self):
+        msg = "Downloading %(pkgno)s / %(total_files)s RPMs, " \
+              "%(downloaded)s / %(total_size)s (%(percent)d%%) done.\n"
+        downloaded = sum(self.downloads.values())
+        vals = {
+            'downloaded'  : downloaded,
+            'percent'     : int(100 * downloaded/self.total_size),
+            'pkgno'       : self.pkgno,
+            'total_files' : self.total_files,
+            'total_size'  : self.total_size
+        }
+        self.output.write(msg % vals)
+
+
[docs]    def end(self, payload, status, err_msg):
+        nevra = str(payload)
+        if status is dnf.callback.STATUS_OK:
+            self.downloads[nevra] = payload.download_size
+            self.pkgno += 1
+            self._update()
+            return
+        logger.critical("Failed to download '%s': %d - %s", nevra, status, err_msg)
+

+
[docs]    def progress(self, payload, done):
+        nevra = str(payload)
+        self.downloads[nevra] = done
+        self._update()
+

+
[docs]    def start(self, total_files, total_size):
+        self.total_files = total_files
+        self.total_size = total_size
+
+

+
[docs]class LoraxRpmCallback(dnf.callback.LoggingTransactionDisplay):
+    def __init__(self, queue):
+        super(LoraxRpmCallback, self).__init__()
+        self._queue = queue
+        self._last_ts = None
+        self.cnt = 0
+
+
[docs]    def event(self, package, action, te_current, te_total, ts_current, ts_total):
+        if action == self.PKG_INSTALL and te_current == 0:
+            # do not report same package twice
+            if self._last_ts == ts_current:
+                return
+            self._last_ts = ts_current
+
+            msg = '(%d/%d) %s.%s' % \
+                (ts_current, ts_total, package.name, package.arch)
+            self.cnt += 1
+            self._queue.put(('install', msg))
+        elif action == self.TRANS_POST:
+            self._queue.put(('post', None))

+
+ +
+ +
+
+ +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/html/_modules/pylorax/executils.html b/docs/html/_modules/pylorax/executils.html new file mode 100644 index 00000000..b23e6b24 --- /dev/null +++ b/docs/html/_modules/pylorax/executils.html @@ -0,0 +1,424 @@ + + + + + + + + + + pylorax.executils — Lorax 23.8 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + +
+ + + + + + +
+
+
+ +
+
+
+ +

Source code for pylorax.executils

+#
+# executil.py - subprocess execution utility functions
+#
+# Copyright (C) 1999-2015
+# Red Hat, Inc.  All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import subprocess
+import signal
+from time import sleep
+
+import logging
+log = logging.getLogger("pylorax")
+program_log = logging.getLogger("program")
+
+from threading import Lock
+program_log_lock = Lock()
+
+_child_env = {}
+
+
[docs]def setenv(name, value):
+    """ Set an environment variable to be used by child processes.
+
+        This method does not modify os.environ for the running process, which
+        is not thread-safe. If setenv has already been called for a particular
+        variable name, the old value is overwritten.
+
+        :param str name: The name of the environment variable
+        :param str value: The value of the environment variable
+    """
+
+    _child_env[name] = value
+

+
[docs]def augmentEnv():
+    env = os.environ.copy()
+    env.update(_child_env)
+    return env
+

+
[docs]class ExecProduct(object):
+    def __init__(self, rc, stdout, stderr):
+        self.rc = rc
+        self.stdout = stdout
+        self.stderr = stderr
+

+
[docs]def startProgram(argv, root='/', stdin=None, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+        env_prune=None, env_add=None, reset_handlers=True, reset_lang=True, **kwargs):
+    """ Start an external program and return the Popen object.
+
+        The root and reset_handlers arguments are handled by passing a
+        preexec_fn argument to subprocess.Popen, but an additional preexec_fn
+        can still be specified and will be run. The user preexec_fn will be run
+        last.
+
+        :param argv: The command to run and argument
+        :param root: The directory to chroot to before running command.
+        :param stdin: The file object to read stdin from.
+        :param stdout: The file object to write stdout to.
+        :param stderr: The file object to write stderr to.
+        :param env_prune: environment variables to remove before execution
+        :param env_add: environment variables to add before execution
+        :param reset_handlers: whether to reset to SIG_DFL any signal handlers set to SIG_IGN
+        :param reset_lang: whether to set the locale of the child process to C
+        :param kwargs: Additional parameters to pass to subprocess.Popen
+        :return: A Popen object for the running command.
+    """
+    if env_prune is None:
+        env_prune = []
+
+    # Check for and save a preexec_fn argument
+    preexec_fn = kwargs.pop("preexec_fn", None)
+
+    def preexec():
+        # If a target root was specificed, chroot into it
+        if root and root != '/':
+            os.chroot(root)
+            os.chdir("/")
+
+        # Signal handlers set to SIG_IGN persist across exec. Reset
+        # these to SIG_DFL if requested. In particular this will include the
+        # SIGPIPE handler set by python.
+        if reset_handlers:
+            for signum in range(1, signal.NSIG):
+                if signal.getsignal(signum) == signal.SIG_IGN:
+                    signal.signal(signum, signal.SIG_DFL)
+
+        # If the user specified an additional preexec_fn argument, run it
+        if preexec_fn is not None:
+            preexec_fn()
+
+    with program_log_lock:
+        program_log.info("Running... %s", " ".join(argv))
+
+    env = augmentEnv()
+    for var in env_prune:
+        env.pop(var, None)
+
+    if reset_lang:
+        env.update({"LC_ALL": "C"})
+
+    if env_add:
+        env.update(env_add)
+
+    return subprocess.Popen(argv,
+                            stdin=stdin,
+                            stdout=stdout,
+                            stderr=stderr,
+                            close_fds=True,
+                            preexec_fn=preexec, cwd=root, env=env, **kwargs)
+

+def _run_program(argv, root='/', stdin=None, stdout=None, env_prune=None, log_output=True,
+        binary_output=False, filter_stderr=False, raise_err=False, callback=None):
+    """ Run an external program, log the output and return it to the caller
+        :param argv: The command to run and argument
+        :param root: The directory to chroot to before running command.
+        :param stdin: The file object to read stdin from.
+        :param stdout: Optional file object to write the output to.
+        :param env_prune: environment variable to remove before execution
+        :param log_output: whether to log the output of command
+        :param binary_output: whether to treat the output of command as binary data
+        :param filter_stderr: whether to exclude the contents of stderr from the returned output
+        :param raise_err: whether to raise a CalledProcessError if the returncode is non-zero
+        :param callback: method to call while waiting for process to finish, passed Popen object
+        :return: The return code of the command and the output
+        :raises: OSError or CalledProcessError
+    """
+    try:
+        if filter_stderr:
+            stderr = subprocess.PIPE
+        else:
+            stderr = subprocess.STDOUT
+
+        proc = startProgram(argv, root=root, stdin=stdin, stdout=subprocess.PIPE, stderr=stderr,
+                            env_prune=env_prune, universal_newlines=not binary_output)
+
+        if callback:
+            while callback(proc) and proc.poll() is None:
+                sleep(1)
+
+        (output_string, err_string) = proc.communicate()
+        if output_string:
+            if binary_output:
+                output_lines = [output_string]
+            else:
+                if output_string[-1] != "\n":
+                    output_string = output_string + "\n"
+                output_lines = output_string.splitlines(True)
+
+            if log_output:
+                with program_log_lock:
+                    for line in output_lines:
+                        program_log.info(line.strip())
+
+            if stdout:
+                stdout.write(output_string)
+
+        # If stderr was filtered, log it separately
+        if filter_stderr and err_string and log_output:
+            err_lines = err_string.splitlines(True)
+
+            with program_log_lock:
+                for line in err_lines:
+                    program_log.info(line.strip())
+
+    except OSError as e:
+        with program_log_lock:
+            program_log.error("Error running %s: %s", argv[0], e.strerror)
+        raise
+
+    with program_log_lock:
+        program_log.debug("Return code: %d", proc.returncode)
+
+    if proc.returncode and raise_err:
+        raise subprocess.CalledProcessError(proc.returncode, argv)
+
+    return (proc.returncode, output_string)
+
+
[docs]def execWithRedirect(command, argv, stdin=None, stdout=None, root='/', env_prune=None,
+                     log_output=True, binary_output=False, raise_err=False, callback=None):
+    """ Run an external program and redirect the output to a file.
+        :param command: The command to run
+        :param argv: The argument list
+        :param stdin: The file object to read stdin from.
+        :param stdout: Optional file object to redirect stdout and stderr to.
+        :param root: The directory to chroot to before running command.
+        :param env_prune: environment variable to remove before execution
+        :param log_output: whether to log the output of command
+        :param binary_output: whether to treat the output of command as binary data
+        :param raise_err: whether to raise a CalledProcessError if the returncode is non-zero
+        :param callback: method to call while waiting for process to finish, passed Popen object
+        :return: The return code of the command
+    """
+    argv = [command] + list(argv)
+    return _run_program(argv, stdin=stdin, stdout=stdout, root=root, env_prune=env_prune,
+            log_output=log_output, binary_output=binary_output, raise_err=raise_err, callback=callback)[0]
+

+
[docs]def execWithCapture(command, argv, stdin=None, root='/', log_output=True, filter_stderr=False,
+                    raise_err=False, callback=None):
+    """ Run an external program and capture standard out and err.
+        :param command: The command to run
+        :param argv: The argument list
+        :param stdin: The file object to read stdin from.
+        :param root: The directory to chroot to before running command.
+        :param log_output: Whether to log the output of command
+        :param filter_stderr: Whether stderr should be excluded from the returned output
+        :param raise_err: whether to raise a CalledProcessError if the returncode is non-zero
+        :return: The output of the command
+    """
+    argv = [command] + list(argv)
+    return _run_program(argv, stdin=stdin, root=root, log_output=log_output, filter_stderr=filter_stderr,
+                        raise_err=raise_err, callback=callback)[1]
+

+
[docs]def runcmd(cmd, **kwargs):
+    """ run execWithRedirect with raise_err=True
+    """
+    kwargs["raise_err"] = True
+    return execWithRedirect(cmd[0], cmd[1:], **kwargs)
+

+
[docs]def runcmd_output(cmd, **kwargs):
+    """ run execWithCapture with raise_err=True
+    """
+    kwargs["raise_err"] = True
+    return execWithCapture(cmd[0], cmd[1:], **kwargs)

+
+ +
+ +
+
+ +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/html/_modules/pylorax/imgutils.html b/docs/html/_modules/pylorax/imgutils.html new file mode 100644 index 00000000..a3ad8b6b --- /dev/null +++ b/docs/html/_modules/pylorax/imgutils.html @@ -0,0 +1,605 @@ + + + + + + + + + + pylorax.imgutils — Lorax 23.8 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + +
+ + + + + + +
+
+
+ +
+
+
+ +

Source code for pylorax.imgutils

+# imgutils.py - utility functions/classes for building disk images
+#
+# Copyright (C) 2011-2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Author(s):  Will Woods <wwoods@redhat.com>
+
+import logging
+logger = logging.getLogger("pylorax.imgutils")
+
+import os, tempfile
+from os.path import join, dirname
+from subprocess import Popen, PIPE, CalledProcessError
+import sys
+import traceback
+import multiprocessing
+from time import sleep
+
+from pylorax.sysutils import cpfile
+from pylorax.executils import execWithRedirect, execWithCapture
+from pylorax.executils import runcmd, runcmd_output
+
+######## Functions for making container images (cpio, tar, squashfs) ##########
+
+
[docs]def compress(command, rootdir, outfile, compression="xz", compressargs=None):
+    '''Make a compressed archive of the given rootdir.
+    command is a list of the archiver commands to run
+    compression should be "xz", "gzip", "lzma", "bzip2", or None.
+    compressargs will be used on the compression commandline.'''
+    if compression not in (None, "xz", "gzip", "lzma", "bzip2"):
+        raise ValueError("Unknown compression type %s" % compression)
+    compressargs = compressargs or ["-9"]
+    if compression == "xz":
+        compressargs.insert(0, "--check=crc32")
+    if compression is None:
+        compression = "cat" # this is a little silly
+        compressargs = []
+
+    # make compression run with multiple threads if possible
+    if compression in ("xz", "lzma"):
+        compressargs.insert(0, "-T%d" % multiprocessing.cpu_count())
+    elif compression == "gzip":
+        compression = "pigz"
+        compressargs.insert(0, "-p%d" % multiprocessing.cpu_count())
+    elif compression == "bzip2":
+        compression = "pbzip2"
+        compressargs.insert(0, "-p%d" % multiprocessing.cpu_count())
+
+    logger.debug("find %s -print0 |%s | %s %s > %s", rootdir, " ".join(command),
+                 compression, " ".join(compressargs), outfile)
+    find, archive, comp = None, None, None
+    try:
+        find = Popen(["find", ".", "-print0"], stdout=PIPE, cwd=rootdir)
+        archive = Popen(command, stdin=find.stdout, stdout=PIPE, cwd=rootdir)
+        comp = Popen([compression] + compressargs,
+                     stdin=archive.stdout, stdout=open(outfile, "wb"))
+        comp.wait()
+        return comp.returncode
+    except OSError as e:
+        logger.error(e)
+        # Kill off any hanging processes
+        list(p.kill() for p in (find, archive, comp) if p)
+        return 1
+

+
[docs]def mkcpio(rootdir, outfile, compression="xz", compressargs=None):
+    compressargs = compressargs or ["-9"]
+    return compress(["cpio", "--null", "--quiet", "-H", "newc", "-o"],
+                    rootdir, outfile, compression, compressargs)
+

+
[docs]def mktar(rootdir, outfile, compression="xz", compressargs=None):
+    compressargs = compressargs or ["-9"]
+    return compress(["tar", "--no-recursion", "--selinux", "--acls", "--xattrs", "-cf-", "--null", "-T-"],
+                    rootdir, outfile, compression, compressargs)
+

+
[docs]def mksquashfs(rootdir, outfile, compression="default", compressargs=None):
+    '''Make a squashfs image containing the given rootdir.'''
+    compressargs = compressargs or []
+    if compression != "default":
+        compressargs = ["-comp", compression] + compressargs
+    return execWithRedirect("mksquashfs", [rootdir, outfile] + compressargs)
+

+
[docs]def mkrootfsimg(rootdir, outfile, label, size=2, sysroot=""):
+    """
+    Make rootfs image from a directory
+
+    :param str rootdir: Root directory
+    :param str outfile: Path of output image file
+    :param str label: Filesystem label
+    :param int size: Size of the image in GiB, if None computed automatically
+    :param str sysroot: path to system (deployment) root relative to physical root
+    """
+    if size:
+        fssize = size * (1024*1024*1024) # 2GB sparse file compresses down to nothin'
+    else:
+        fssize = None       # Let mkext4img figure out the needed size
+
+    mkext4img(rootdir, outfile, label=label, size=fssize)
+    # Reset selinux context on new rootfs
+    with LoopDev(outfile) as loopdev:
+        with Mount(loopdev) as mnt:
+            cmd = [ "setfiles", "-e", "/proc", "-e", "/sys", "-e", "/dev", "-e", "/install",
+                    "/etc/selinux/targeted/contexts/files/file_contexts", "/"]
+            root = join(mnt, sysroot.lstrip("/"))
+            runcmd(cmd, root=root)
+
+######## Utility functions ###############################################
+

+
[docs]def mksparse(outfile, size):
+    '''use os.ftruncate to create a sparse file of the given size.'''
+    fobj = open(outfile, "w")
+    os.ftruncate(fobj.fileno(), size)
+

+
[docs]def mkqcow2(outfile, size, options=None):
+    '''use qemu-img to create a file of the given size.
+       options is a list of options passed to qemu-img
+
+       Default format is qcow2, override by passing "-f", fmt
+       in options.
+    '''
+    options = options or []
+    if "-f" not in options:
+        options.extend(["-f", "qcow2"])
+    runcmd(["qemu-img", "create"] + options + [outfile, str(size)])
+

+
[docs]def loop_attach(outfile):
+    '''Attach a loop device to the given file. Return the loop device name.
+    Raises CalledProcessError if losetup fails.'''
+    dev = runcmd_output(["losetup", "--find", "--show", outfile])
+    return dev.strip()
+

+
[docs]def loop_detach(loopdev):
+    '''Detach the given loop device. Return False on failure.'''
+    return (execWithRedirect("losetup", ["--detach", loopdev]) == 0)
+

+
[docs]def get_loop_name(path):
+    '''Return the loop device associated with the path.
+    Raises RuntimeError if more than one loop is associated'''
+    buf = runcmd_output(["losetup", "-j", path])
+    if len(buf.splitlines()) > 1:
+        # there should never be more than one loop device listed
+        raise RuntimeError("multiple loops associated with %s" % path)
+    name = os.path.basename(buf.split(":")[0])
+    return name
+

+
[docs]def dm_attach(dev, size, name=None):
+    '''Attach a devicemapper device to the given device, with the given size.
+    If name is None, a random name will be chosen. Returns the device name.
+    raises CalledProcessError if dmsetup fails.'''
+    if name is None:
+        name = tempfile.mktemp(prefix="lorax.imgutils.", dir="")
+    runcmd(["dmsetup", "create", name, "--table",
+                       "0 %i linear %s 0" % (size/512, dev)])
+    return name
+

+
[docs]def dm_detach(dev):
+    '''Detach the named devicemapper device. Returns False if dmsetup fails.'''
+    dev = dev.replace("/dev/mapper/", "") # strip prefix, if it's there
+    return execWithRedirect("dmsetup", ["remove", dev])
+

+
[docs]def mount(dev, opts="", mnt=None):
+    '''Mount the given device at the given mountpoint, using the given opts.
+    opts should be a comma-separated string of mount options.
+    if mnt is none, a temporary directory will be created and its path will be
+    returned.
+    raises CalledProcessError if mount fails.'''
+    if mnt is None:
+        mnt = tempfile.mkdtemp(prefix="lorax.imgutils.")
+        logger.debug("make tmp mountdir %s", mnt)
+    cmd = ["mount"]
+    if opts:
+        cmd += ["-o", opts]
+    cmd += [dev, mnt]
+    runcmd(cmd)
+    return mnt
+

+
[docs]def umount(mnt,  lazy=False, maxretry=3, retrysleep=1.0):
+    '''Unmount the given mountpoint. If lazy is True, do a lazy umount (-l).
+    If the mount was a temporary dir created by mount, it will be deleted.
+    raises CalledProcessError if umount fails.'''
+    cmd = ["umount"]
+    if lazy: cmd += ["-l"]
+    cmd += [mnt]
+    count = 0
+    while maxretry > 0:
+        try:
+            rv = runcmd(cmd)
+        except CalledProcessError:
+            count += 1
+            if count == maxretry:
+                raise
+            logger.warn("failed to unmount %s. retrying (%d/%d)...",
+                         mnt, count, maxretry)
+            if logger.getEffectiveLevel() <= logging.DEBUG:
+                fuser = execWithCapture("fuser", ["-vm", mnt])
+                logger.debug("fuser -vm:\n%s\n", fuser)
+            sleep(retrysleep)
+        else:
+            break
+    if 'lorax.imgutils' in mnt:
+        os.rmdir(mnt)
+        logger.debug("remove tmp mountdir %s", mnt)
+    return (rv == 0)
+

+
[docs]def copytree(src, dest, preserve=True):
+    '''Copy a tree of files using cp -a, thus preserving modes, timestamps,
+    links, acls, sparse files, xattrs, selinux contexts, etc.
+    If preserve is False, uses cp -R (useful for modeless filesystems)
+    raises CalledProcessError if copy fails.'''
+    logger.debug("copytree %s %s", src, dest)
+    cp = ["cp", "-a"] if preserve else ["cp", "-R", "-L"]
+    cp += [join(src, "."), os.path.abspath(dest)]
+    runcmd(cp)
+

+
[docs]def do_grafts(grafts, dest, preserve=True):
+    '''Copy each of the items listed in grafts into dest.
+    If the key ends with '/' it's assumed to be a directory which should be
+    created, otherwise just the leading directories will be created.'''
+    for imgpath, filename in grafts.items():
+        if imgpath[-1] == '/':
+            targetdir = join(dest, imgpath)
+            imgpath = imgpath[:-1]
+        else:
+            targetdir = join(dest, dirname(imgpath))
+        if not os.path.isdir(targetdir):
+            os.makedirs(targetdir)
+        if os.path.isdir(filename):
+            copytree(filename, join(dest, imgpath), preserve)
+        else:
+            cpfile(filename, join(dest, imgpath))
+

+
[docs]def round_to_blocks(size, blocksize):
+    '''If size isn't a multiple of blocksize, round up to the next multiple'''
+    diff = size % blocksize
+    if diff or not size:
+        size += blocksize - diff
+    return size
+
+# TODO: move filesystem data outside this function

+
[docs]def estimate_size(rootdir, graft=None, fstype=None, blocksize=4096, overhead=128):
+    graft = graft or {}
+    getsize = lambda f: os.lstat(f).st_size
+    if fstype == "btrfs":
+        overhead = 64*1024 # don't worry, it's all sparse
+    if fstype == "hfsplus":
+        overhead = 200 # hack to deal with two bootloader copies
+    if fstype in ("vfat", "msdos"):
+        blocksize = 2048
+        getsize = lambda f: os.stat(f).st_size # no symlinks, count as copies
+    total = overhead*blocksize
+    dirlist = list(graft.values())
+    if rootdir:
+        dirlist.append(rootdir)
+    for root in dirlist:
+        for top, dirs, files in os.walk(root):
+            for f in files + dirs:
+                total += round_to_blocks(getsize(join(top,f)), blocksize)
+    if fstype == "btrfs":
+        total = max(256*1024*1024, total) # btrfs minimum size: 256MB
+    return total
+
+######## Execution contexts - use with the 'with' statement ##############
+

+
[docs]class LoopDev(object):
+    def __init__(self, filename, size=None):
+        self.loopdev = None
+        self.filename = filename
+        if size:
+            mksparse(self.filename, size)
+    def __enter__(self):
+        self.loopdev = loop_attach(self.filename)
+        return self.loopdev
+    def __exit__(self, exc_type, exc_value, tracebk):
+        loop_detach(self.loopdev)
+

+
[docs]class DMDev(object):
+    def __init__(self, dev, size, name=None):
+        self.mapperdev = None
+        (self.dev, self.size, self.name) = (dev, size, name)
+    def __enter__(self):
+        self.mapperdev = dm_attach(self.dev, self.size, self.name)
+        return self.mapperdev
+    def __exit__(self, exc_type, exc_value, tracebk):
+        dm_detach(self.mapperdev)
+

+
[docs]class Mount(object):
+    def __init__(self, dev, opts="", mnt=None):
+        (self.dev, self.opts, self.mnt) = (dev, opts, mnt)
+    def __enter__(self):
+        self.mnt = mount(self.dev, self.opts, self.mnt)
+        return self.mnt
+    def __exit__(self, exc_type, exc_value, tracebk):
+        umount(self.mnt)
+

+
[docs]class PartitionMount(object):
+    """ Mount a partitioned image file using kpartx """
+    def __init__(self, disk_img, mount_ok=None):
+        """
+        disk_img is the full path to a partitioned disk image
+        mount_ok is a function that is passed the mount point and
+        returns True if it should be mounted.
+        """
+        self.mount_dev = None
+        self.mount_size = None
+        self.mount_dir = None
+        self.disk_img = disk_img
+        self.mount_ok = mount_ok
+
+        # Default is to mount partition with /etc/passwd
+        if not self.mount_ok:
+            self.mount_ok = lambda mount_dir: os.path.isfile(mount_dir+"/etc/passwd")
+
+        # Example kpartx output
+        # kpartx -p p -v -a /tmp/diskV2DiCW.im
+        # add map loop2p1 (253:2): 0 3481600 linear /dev/loop2 2048
+        # add map loop2p2 (253:3): 0 614400 linear /dev/loop2 3483648
+        kpartx_output = runcmd_output(["kpartx", "-v", "-a", "-s", self.disk_img])
+        logger.debug(kpartx_output)
+
+        # list of (deviceName, sizeInBytes)
+        self.loop_devices = []
+        for line in kpartx_output.splitlines():
+            # add map loop2p3 (253:4): 0 7139328 linear /dev/loop2 528384
+            # 3rd element is size in 512 byte blocks
+            if line.startswith("add map "):
+                fields = line[8:].split()
+                self.loop_devices.append( (fields[0], int(fields[3])*512) )
+
+    def __enter__(self):
+        # Mount the device selected by mount_ok, if possible
+        mount_dir = tempfile.mkdtemp()
+        for dev, size in self.loop_devices:
+            try:
+                mount( "/dev/mapper/"+dev, mnt=mount_dir )
+                if self.mount_ok(mount_dir):
+                    self.mount_dir = mount_dir
+                    self.mount_dev = dev
+                    self.mount_size = size
+                    break
+                umount( mount_dir )
+            except CalledProcessError:
+                logger.debug(traceback.format_exc())
+        if self.mount_dir:
+            logger.info("Partition mounted on %s size=%s", self.mount_dir, self.mount_size)
+        else:
+            logger.debug("Unable to mount anything from %s", self.disk_img)
+            os.rmdir(mount_dir)
+        return self
+
+    def __exit__(self, exc_type, exc_value, tracebk):
+        if self.mount_dir:
+            umount( self.mount_dir )
+            os.rmdir(self.mount_dir)
+            self.mount_dir = None
+        execWithRedirect("kpartx", ["-d", "-s", self.disk_img])
+
+
+######## Functions for making filesystem images ##########################
+

+
[docs]def mkfsimage(fstype, rootdir, outfile, size=None, mkfsargs=None, mountargs="", graft=None):
+    '''Generic filesystem image creation function.
+    fstype should be a filesystem type - "mkfs.${fstype}" must exist.
+    graft should be a dict: {"some/path/in/image": "local/file/or/dir"};
+      if the path ends with a '/' it's assumed to be a directory.
+    Will raise CalledProcessError if something goes wrong.'''
+    mkfsargs = mkfsargs or []
+    graft = graft or {}
+    preserve = (fstype not in ("msdos", "vfat"))
+    if not size:
+        size = estimate_size(rootdir, graft, fstype)
+    with LoopDev(outfile, size) as loopdev:
+        try:
+            runcmd(["mkfs.%s" % fstype] + mkfsargs + [loopdev])
+        except CalledProcessError as e:
+            logger.error("mkfs exited with a non-zero return code: %d", e.returncode)
+            logger.error(e.output)
+            sys.exit(e.returncode)
+
+        with Mount(loopdev, mountargs) as mnt:
+            if rootdir:
+                copytree(rootdir, mnt, preserve)
+            do_grafts(graft, mnt, preserve)
+
+    # Make absolutely sure that the data has been written
+    runcmd(["sync"])
+
+# convenience functions with useful defaults

+
[docs]def mkdosimg(rootdir, outfile, size=None, label="", mountargs="shortname=winnt,umask=0077", graft=None):
+    graft = graft or {}
+    mkfsimage("msdos", rootdir, outfile, size, mountargs=mountargs,
+              mkfsargs=["-n", label], graft=graft)
+

+
[docs]def mkext4img(rootdir, outfile, size=None, label="", mountargs="", graft=None):
+    graft = graft or {}
+    mkfsimage("ext4", rootdir, outfile, size, mountargs=mountargs,
+              mkfsargs=["-L", label, "-b", "1024", "-m", "0"], graft=graft)
+

+
[docs]def mkbtrfsimg(rootdir, outfile, size=None, label="", mountargs="", graft=None):
+    graft = graft or {}
+    mkfsimage("btrfs", rootdir, outfile, size, mountargs=mountargs,
+               mkfsargs=["-L", label], graft=graft)
+

+
[docs]def mkhfsimg(rootdir, outfile, size=None, label="", mountargs="", graft=None):
+    graft = graft or {}
+    mkfsimage("hfsplus", rootdir, outfile, size, mountargs=mountargs,
+              mkfsargs=["-v", label], graft=graft)

+
+ +
+ +
+
+ +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/html/_modules/pylorax/ltmpl.html b/docs/html/_modules/pylorax/ltmpl.html new file mode 100644 index 00000000..30423537 --- /dev/null +++ b/docs/html/_modules/pylorax/ltmpl.html @@ -0,0 +1,902 @@ + + + + + + + + + + pylorax.ltmpl — Lorax 23.8 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + +
+ + + + + + +
+
+
+ +
+
+
+ +

Source code for pylorax.ltmpl

+#
+# ltmpl.py
+#
+# Copyright (C) 2009-2015  Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Red Hat Author(s):  Martin Gracik <mgracik@redhat.com>
+#                     Will Woods <wwoods@redhat.com>
+#
+
+import logging
+logger = logging.getLogger("pylorax.ltmpl")
+
+import os, re, glob, shlex, fnmatch
+from os.path import basename, isdir
+from subprocess import CalledProcessError
+
+from pylorax.sysutils import joinpaths, cpfile, mvfile, replace, remove
+from pylorax.dnfhelper import LoraxDownloadCallback, LoraxRpmCallback
+from pylorax.base import DataHolder
+from pylorax.executils import runcmd, runcmd_output
+from pylorax.imgutils import mkcpio
+import pylorax.output as output
+
+from mako.lookup import TemplateLookup
+from mako.exceptions import text_error_template
+import sys, traceback
+import struct
+import dnf
+import multiprocessing
+import queue
+import collections
+
+
[docs]class LoraxTemplate(object):
+    def __init__(self, directories=None):
+        directories = directories or ["/usr/share/lorax"]
+        # we have to add ["/"] to the template lookup directories or the
+        # file includes won't work properly for absolute paths
+        self.directories = ["/"] + directories
+
+
[docs]    def parse(self, template_file, variables):
+        lookup = TemplateLookup(directories=self.directories)
+        template = lookup.get_template(template_file)
+
+        try:
+            textbuf = template.render(**variables)
+        except:
+            logger.error("Problem rendering %s (%s):", template_file, variables)
+            logger.error(text_error_template().render())
+            raise
+
+        # split, strip and remove empty lines
+        lines = textbuf.splitlines()
+        lines = [line.strip() for line in lines]
+        lines = [line for line in lines if line]
+
+        # remove comments
+        lines = [line for line in lines if not line.startswith("#")]
+
+        # split with shlex and perform brace expansion
+        lines = [split_and_expand(line) for line in lines]
+
+        return lines
+

+
[docs]def split_and_expand(line):
+    return [exp for word in shlex.split(line) for exp in brace_expand(word)]
+

+
[docs]def brace_expand(s):
+    if not ('{' in s and ',' in s and '}' in s):
+        yield s
+    else:
+        right = s.find('}')
+        left = s[:right].rfind('{')
+        (prefix, choices, suffix) = (s[:left], s[left+1:right], s[right+1:])
+        for choice in choices.split(','):
+            for alt in brace_expand(prefix+choice+suffix):
+                yield alt
+

+
[docs]def rglob(pathname, root="/", fatal=False):
+    seen = set()
+    rootlen = len(root)+1
+    for f in glob.iglob(joinpaths(root, pathname)):
+        if f not in seen:
+            seen.add(f)
+            yield f[rootlen:] # remove the root to produce relative path
+    if fatal and not seen:
+        raise IOError("nothing matching %s in %s" % (pathname, root))
+

+
[docs]def rexists(pathname, root=""):
+    # Generator is always True, even with no values;
+    # bool(rglob(...)) won't work here.
+    for _path in rglob(pathname, root):
+        return True
+    return False
+
+# TODO: operate inside an actual chroot for safety? Not that RPM bothers..

+
[docs]class LoraxTemplateRunner(object):
+    '''
+    This class parses and executes Lorax templates. Sample usage:
+
+      # install a bunch of packages
+      runner = LoraxTemplateRunner(inroot=rundir, outroot=rundir, dbo=dnf_obj)
+      runner.run("install-packages.ltmpl")
+
+      # modify a runtime dir
+      runner = LoraxTemplateRunner(inroot=rundir, outroot=newrun)
+      runner.run("runtime-transmogrify.ltmpl")
+
+    NOTES:
+
+    * Parsing procedure is roughly:
+      1. Mako template expansion (on the whole file)
+      2. For each line of the result,
+        a. Whitespace splitting (using shlex.split())
+        b. Brace expansion (using brace_expand())
+        c. If the first token is the name of a function, call that function
+           with the rest of the line as arguments
+
+    * Parsing and execution are *separate* passes - so you can't use the result
+      of a command in an %if statement (or any other control statements)!
+
+    * Commands that run external programs (systemctl, gconfset) currently use
+      the *host*'s copy of that program, which may cause problems if there's a
+      big enough difference between the host and the image you're modifying.
+
+    * The commands are not executed under a real chroot, so absolute symlinks
+      will point *outside* the inroot/outroot. Be careful with symlinks!
+
+    ADDING NEW COMMANDS:
+
+    * Each template command is just a method of the LoraxTemplateRunner
+      object - so adding a new command is as easy as adding a new function.
+
+    * Each function gets arguments that correspond to the rest of the tokens
+      on that line (after word splitting and brace expansion)
+
+    * Commands should raise exceptions for errors - don't use sys.exit()
+    '''
+    def __init__(self, inroot, outroot, dbo=None, fatalerrors=True,
+                                        templatedir=None, defaults=None):
+        self.inroot = inroot
+        self.outroot = outroot
+        self.dbo = dbo
+        self.fatalerrors = fatalerrors
+        self.templatedir = templatedir or "/usr/share/lorax"
+        self.templatefile = None
+        # some builtin methods
+        self.builtins = DataHolder(exists=lambda p: rexists(p, root=inroot),
+                                   glob=lambda g: list(rglob(g, root=inroot)))
+        self.defaults = defaults or {}
+        self.results = DataHolder(treeinfo=dict()) # just treeinfo for now
+        # TODO: set up custom logger with a filter to add line info
+
+    def _out(self, path):
+        return joinpaths(self.outroot, path)
+    def _in(self, path):
+        return joinpaths(self.inroot, path)
+
+    def _filelist(self, *pkgs):
+        """ Return the list of files in the packages """
+        pkglist = []
+        for pkg_glob in pkgs:
+            pkglist += list(self.dbo.sack.query().installed().filter(name__glob=pkg_glob))
+
+        # dnf/hawkey doesn't make any distinction between file, dir or ghost like yum did
+        # so only return the files.
+        return set(f for pkg in pkglist for f in pkg.files if not os.path.isdir(self._out(f)))
+
+    def _getsize(self, *files):
+        return sum(os.path.getsize(self._out(f)) for f in files if os.path.isfile(self._out(f)))
+
+
[docs]    def run(self, templatefile, **variables):
+        for k,v in list(self.defaults.items()) + list(self.builtins.items()):
+            variables.setdefault(k,v)
+        logger.debug("executing %s with variables=%s", templatefile, variables)
+        self.templatefile = templatefile
+        t = LoraxTemplate(directories=[self.templatedir])
+        commands = t.parse(templatefile, variables)
+        self._run(commands)
+
+

+    def _run(self, parsed_template):
+        logger.info("running %s", self.templatefile)
+        for (num, line) in enumerate(parsed_template,1):
+            logger.debug("template line %i: %s", num, " ".join(line))
+            skiperror = False
+            (cmd, args) = (line[0], line[1:])
+            # Following Makefile convention, if the command is prefixed with
+            # a dash ('-'), we'll ignore any errors on that line.
+            if cmd.startswith('-'):
+                cmd = cmd[1:]
+                skiperror = True
+            try:
+                # grab the method named in cmd and pass it the given arguments
+                f = getattr(self, cmd, None)
+                if cmd[0] == '_' or cmd == 'run' or not isinstance(f, collections.Callable):
+                    raise ValueError("unknown command %s" % cmd)
+                f(*args)
+            except Exception: # pylint: disable=broad-except
+                if skiperror:
+                    logger.debug("ignoring error")
+                    continue
+                logger.error("template command error in %s:", self.templatefile)
+                logger.error("  %s", " ".join(line))
+                # format the exception traceback
+                exclines = traceback.format_exception(*sys.exc_info())
+                # skip the bit about "ltmpl.py, in _run()" - we know that
+                exclines.pop(1)
+                # log the "ErrorType: this is what happened" line
+                logger.error("  " + exclines[-1].strip())
+                # and log the entire traceback to the debug log
+                for line in ''.join(exclines).splitlines():
+                    logger.debug("  " + line)
+                if self.fatalerrors:
+                    raise
+
+
[docs]    def install(self, srcglob, dest):
+        '''
+        install SRC DEST
+          Copy the given file (or files, if a glob is used) from the input
+          tree to the given destination in the output tree.
+          The path to DEST must exist in the output tree.
+          If DEST is a directory, SRC will be copied into that directory.
+          If DEST doesn't exist, SRC will be copied to a file with that name,
+          assuming the rest of the path exists.
+          This is pretty much like how the 'cp' command works.
+          Examples:
+            install usr/share/myconfig/grub.conf /boot
+            install /usr/share/myconfig/grub.conf.in /boot/grub.conf
+        '''
+        for src in rglob(self._in(srcglob), fatal=True):
+            cpfile(src, self._out(dest))
+

+
[docs]    def installimg(self, srcdir, destfile):
+        '''
+        installimg SRCDIR DESTFILE
+          Create a compressed cpio archive of the contents of SRCDIR and place
+          it in DESTFILE.
+
+          If SRCDIR doesn't exist or is empty nothing is created.
+
+          Examples:
+            installimg ${LORAXDIR}/product/ images/product.img
+            installimg ${LORAXDIR}/updates/ images/updates.img
+        '''
+        if not os.path.isdir(self._in(srcdir)) or not os.listdir(self._in(srcdir)):
+            return
+        logger.info("Creating image file %s from contents of %s", self._out(destfile), self._in(srcdir))
+        mkcpio(self._in(srcdir), self._out(destfile))
+

+
[docs]    def mkdir(self, *dirs):
+        '''
+        mkdir DIR [DIR ...]
+          Create the named DIR(s). Will create leading directories as needed.
+          Example:
+            mkdir /images
+        '''
+        for d in dirs:
+            d = self._out(d)
+            if not isdir(d):
+                os.makedirs(d)
+

+
[docs]    def replace(self, pat, repl, *fileglobs):
+        '''
+        replace PATTERN REPLACEMENT FILEGLOB [FILEGLOB ...]
+          Find-and-replace the given PATTERN (Python-style regex) with the given
+          REPLACEMENT string for each of the files listed.
+          Example:
+            replace @VERSION@ ${product.version} /boot/grub.conf /boot/isolinux.cfg
+        '''
+        match = False
+        for g in fileglobs:
+            for f in rglob(self._out(g)):
+                match = True
+                replace(f, pat, repl)
+        if not match:
+            raise IOError("no files matched %s" % " ".join(fileglobs))
+

+
[docs]    def append(self, filename, data):
+        '''
+        append FILE STRING
+          Append STRING (followed by a newline character) to FILE.
+          Python character escape sequences ('\\n', '\\t', etc.) will be
+          converted to the appropriate characters.
+          Examples:
+            append /etc/depmod.d/dd.conf "search updates built-in"
+            append /etc/resolv.conf ""
+        '''
+        with open(self._out(filename), "a") as fobj:
+            fobj.write(bytes(data, "utf8").decode('unicode_escape')+"\n")
+

+
[docs]    def treeinfo(self, section, key, *valuetoks):
+        '''
+        treeinfo SECTION KEY ARG [ARG ...]
+          Add an item to the treeinfo data store.
+          The given SECTION will have a new item added where
+          KEY = ARG ARG ...
+          Example:
+            treeinfo images-${kernel.arch} boot.iso images/boot.iso
+        '''
+        if section not in self.results.treeinfo:
+            self.results.treeinfo[section] = dict()
+        self.results.treeinfo[section][key] = " ".join(valuetoks)
+

+
[docs]    def installkernel(self, section, src, dest):
+        '''
+        installkernel SECTION SRC DEST
+          Install the kernel from SRC in the input tree to DEST in the output
+          tree, and then add an item to the treeinfo data store, in the named
+          SECTION, where "kernel" = DEST.
+
+          Equivalent to:
+            install SRC DEST
+            treeinfo SECTION kernel DEST
+        '''
+        self.install(src, dest)
+        self.treeinfo(section, "kernel", dest)
+

+
[docs]    def installinitrd(self, section, src, dest):
+        '''
+        installinitrd SECTION SRC DEST
+          Same as installkernel, but for "initrd".
+        '''
+        self.install(src, dest)
+        self.chmod(dest, '644')
+        self.treeinfo(section, "initrd", dest)
+

+
[docs]    def installupgradeinitrd(self, section, src, dest):
+        '''
+        installupgradeinitrd SECTION SRC DEST
+          Same as installkernel, but for "upgrade".
+        '''
+        self.install(src, dest)
+        self.chmod(dest, '644')
+        self.treeinfo(section, "upgrade", dest)
+

+
+
+
[docs]    def copy(self, src, dest):
+        '''
+        copy SRC DEST
+          Copy SRC to DEST.
+          If DEST is a directory, SRC will be copied inside it.
+          If DEST doesn't exist, SRC will be copied to a file with
+          that name, if the path leading to it exists.
+        '''
+        cpfile(self._out(src), self._out(dest))
+

+
[docs]    def move(self, src, dest):
+        '''
+        move SRC DEST
+          Move SRC to DEST.
+        '''
+        mvfile(self._out(src), self._out(dest))
+

+
[docs]    def remove(self, *fileglobs):
+        '''
+        remove FILEGLOB [FILEGLOB ...]
+          Remove all the named files or directories.
+          Will *not* raise exceptions if the file(s) are not found.
+        '''
+        for g in fileglobs:
+            for f in rglob(self._out(g)):
+                remove(f)
+                logger.debug("removed %s", f)
+

+
[docs]    def chmod(self, fileglob, mode):
+        '''
+        chmod FILEGLOB OCTALMODE
+          Change the mode of all the files matching FILEGLOB to OCTALMODE.
+        '''
+        for f in rglob(self._out(fileglob), fatal=True):
+            os.chmod(f, int(mode,8))
+
+    # TODO: do we need a new command for gsettings?

+
[docs]    def gconfset(self, path, keytype, value, outfile=None):
+        '''
+        gconfset PATH KEYTYPE VALUE [OUTFILE]
+          Set the given gconf PATH, with type KEYTYPE, to the given value.
+          OUTFILE defaults to /etc/gconf/gconf.xml.defaults if not given.
+          Example:
+            gconfset /apps/metacity/general/num_workspaces int 1
+        '''
+        if outfile is None:
+            outfile = self._out("etc/gconf/gconf.xml.defaults")
+        cmd = ["gconftool-2", "--direct",
+                    "--config-source=xml:readwrite:%s" % outfile,
+                    "--set", "--type", keytype, path, value]
+        runcmd(cmd)
+

+
[docs]    def log(self, msg):
+        '''
+        log MESSAGE
+          Emit the given log message. Be sure to put it in quotes!
+          Example:
+            log "Reticulating splines, please wait..."
+        '''
+        logger.info(msg)
+
+    # TODO: add ssh-keygen, mkisofs(?), find, and other useful commands

+
[docs]    def runcmd(self, *cmdlist):
+        '''
+        runcmd CMD [ARG ...]
+          Run the given command with the given arguments.
+
+          NOTE: All paths given MUST be COMPLETE, ABSOLUTE PATHS to the file
+          or files mentioned. ${root}/${inroot}/${outroot} are good for
+          constructing these paths.
+
+          FURTHER NOTE: Please use this command only as a last resort!
+          Whenever possible, you should use the existing template commands.
+          If the existing commands don't do what you need, fix them!
+
+          Examples:
+            (this should be replaced with a "find" function)
+            runcmd find ${root} -name "*.pyo" -type f -delete
+            %for f in find(root, name="*.pyo"):
+                remove ${f}
+            %endfor
+        '''
+        cmd = cmdlist
+        logger.debug('running command: %s', cmd)
+        if cmd[0].startswith("--chdir="):
+            logger.error("--chdir is no longer supported for runcmd.")
+            raise ValueError("--chdir is no longer supported for runcmd.")
+
+        try:
+            stdout = runcmd_output(cmd)
+            if stdout:
+                logger.debug('command output:\n%s', stdout)
+            logger.debug("command finished successfully")
+        except CalledProcessError as e:
+            if e.output:
+                logger.debug('command output:\n%s', e.output)
+            logger.debug('command returned failure (%d)', e.returncode)
+            raise
+

+
[docs]    def installpkg(self, *pkgs):
+        '''
+        installpkg [--required] PKGGLOB [PKGGLOB ...]
+          Request installation of all packages matching the given globs.
+          Note that this is just a *request* - nothing is *actually* installed
+          until the 'run_pkg_transaction' command is given.
+        '''
+        required = False
+        if pkgs[0] == '--required':
+            pkgs = pkgs[1:]
+            required = True
+
+        for p in pkgs:
+            try:
+                self.dbo.install(p)
+            except Exception as e: # pylint: disable=broad-except
+                # FIXME: save exception and re-raise after the loop finishes
+                logger.error("installpkg %s failed: %s", p, str(e))
+                if required:
+                    raise
+

+
[docs]    def removepkg(self, *pkgs):
+        '''
+        removepkg PKGGLOB [PKGGLOB...]
+          Delete the named package(s).
+          IMPLEMENTATION NOTES:
+            RPM scriptlets (%preun/%postun) are *not* run.
+            Files are deleted, but directories are left behind.
+        '''
+        for p in pkgs:
+            filepaths = [f.lstrip('/') for f in self._filelist(p)]
+            # TODO: also remove directories that aren't owned by anything else
+            if filepaths:
+                logger.debug("removepkg %s: %ikb", p, self._getsize(*filepaths)/1024)
+                self.remove(*filepaths)
+            else:
+                logger.debug("removepkg %s: no files to remove!", p)
+

+
[docs]    def get_token_checked(self, process, token_queue):
+        """Try to get token from queue checking that process is still alive"""
+
+        try:
+            # wait at most a minute for the token
+            (token, msg) = token_queue.get(timeout=60)
+        except queue.Empty:
+            if process.is_alive():
+                try:
+                    # process still alive, give it 2 minutes more
+                    (token, msg) = token_queue.get(timeout=120)
+                except queue.Empty:
+                    # waited for 3 minutes and got nothing
+                    raise Exception("The transaction process got stuck somewhere (no message from it in 3 minutes)")
+            else:
+                raise Exception("The transaction process has ended abruptly")
+
+        return (token, msg)
+

+
[docs]    def run_pkg_transaction(self):
+        '''
+        run_pkg_transaction
+          Actually install all the packages requested by previous 'installpkg'
+          commands.
+        '''
+
+        def do_transaction(base, token_queue):
+            try:
+                display = LoraxRpmCallback(token_queue)
+                base.do_transaction(display=display)
+            except BaseException as e:
+                logger.error("The transaction process has ended abruptly: %s", e)
+                token_queue.put(('quit', str(e)))
+
+        try:
+            logger.info("Checking dependencies")
+            self.dbo.resolve()
+        except dnf.exceptions.DepsolveError as e:
+            logger.error("Dependency check failed: %s", e)
+            raise
+        logger.info("%d packages selected", len(self.dbo.transaction))
+        if len(self.dbo.transaction) == 0:
+            raise Exception("No packages in transaction")
+
+        pkgs_to_download = self.dbo.transaction.install_set
+        logger.info("Downloading packages")
+        progress = LoraxDownloadCallback()
+        try:
+            self.dbo.download_packages(pkgs_to_download, progress)
+        except dnf.exceptions.DownloadError as e:
+            logger.error("Failed to download the following packages: %s", e)
+            raise
+
+        logger.info("Preparing transaction from installation source")
+        token_queue = multiprocessing.Queue()
+        msgout = output.LoraxOutput()
+        process = multiprocessing.Process(target=do_transaction, args=(self.dbo, token_queue))
+        process.start()
+        (token, msg) = self.get_token_checked(process, token_queue)
+
+        while token not in ('post', 'quit'):
+            if token == 'install':
+                logging.info("%s", msg)
+                msgout.writeline(msg)
+            (token, msg) = self.get_token_checked(process, token_queue)
+
+        if token == 'quit':
+            logger.error("Transaction failed.")
+            raise Exception("Transaction failed")
+
+        logger.info("Performing post-installation setup tasks")
+        process.join()
+
+        # Reset the package sack to pick up the installed packages
+        self.dbo.reset(repos=False)
+        self.dbo.fill_sack(load_system_repo=True, load_available_repos=False)
+
+        # At this point dnf should know about the installed files. Double check that it really does.
+        if len(self._filelist("anaconda-core")) == 0:
+            raise Exception("Failed to reset dbo to installed package set")
+

+
[docs]    def removefrom(self, pkg, *globs):
+        '''
+        removefrom PKGGLOB [--allbut] FILEGLOB [FILEGLOB...]
+          Remove all files matching the given file globs from the package
+          (or packages) named.
+          If '--allbut' is used, all the files from the given package(s) will
+          be removed *except* the ones which match the file globs.
+          Examples:
+            removefrom usbutils /usr/bin/*
+            removefrom xfsprogs --allbut /sbin/*
+        '''
+        cmd = "%s %s" % (pkg, " ".join(globs)) # save for later logging
+        keepmatches = False
+        if globs[0] == '--allbut':
+            keepmatches = True
+            globs = globs[1:]
+        # get pkg filelist and find files that match the globs
+        filelist = self._filelist(pkg)
+        matches = set()
+        for g in globs:
+            globs_re = re.compile(fnmatch.translate(g))
+            m = [f for f in filelist if globs_re.match(f)]
+            if m:
+                matches.update(m)
+            else:
+                logger.debug("removefrom %s %s: no files matched!", pkg, g)
+        # are we removing the matches, or keeping only the matches?
+        if keepmatches:
+            remove_files = filelist.difference(matches)
+        else:
+            remove_files = matches
+        # remove the files
+        if remove_files:
+            logger.debug("removefrom %s: removed %i/%i files, %ikb/%ikb", cmd,
+                             len(remove_files), len(filelist),
+                             self._getsize(*remove_files)/1024, self._getsize(*filelist)/1024)
+            self.remove(*remove_files)
+        else:
+            logger.debug("removefrom %s: no files to remove!", cmd)
+

+
[docs]    def removekmod(self, *globs):
+        '''
+        removekmod GLOB [GLOB...] [--allbut] KEEPGLOB [KEEPGLOB...]
+          Remove all files and directories matching the given file globs from the kernel
+          modules directory.
+
+          If '--allbut' is used, all the files from the modules will be removed *except*
+          the ones which match the file globs. There must be at least one initial GLOB
+          to search and one KEEPGLOB to keep. The KEEPGLOB is expanded to be *KEEPGLOB*
+          so that it will match anywhere in the path.
+
+          This only removes files from under /lib/modules/*/kernel/
+
+          Examples:
+            removekmod sound drivers/media drivers/hwmon drivers/video
+            removekmod drivers/char --allbut virtio_console hw_random
+        '''
+        cmd = " ".join(globs)
+        if "--allbut" in globs:
+            idx = globs.index("--allbut")
+            if idx == 0:
+                raise ValueError("removekmod needs at least one GLOB before --allbut")
+
+            # Apply keepglobs anywhere they appear in the path
+            keepglobs = globs[idx+1:]
+            if len(keepglobs) == 0:
+                raise ValueError("removekmod needs at least one GLOB after --allbut")
+
+            globs = globs[:idx]
+        else:
+            # Nothing to keep
+            keepglobs = []
+
+        filelist = set()
+        for g in globs:
+            for top_dir in rglob(self._out("/lib/modules/*/kernel/"+g)):
+                for root, _dirs, files in os.walk(top_dir):
+                    filelist.update(root+"/"+f for f in files)
+
+        # Remove anything matching keepglobs from the list
+        matches = set()
+        for g in keepglobs:
+            globs_re = re.compile(fnmatch.translate("*"+g+"*"))
+            m = [f for f in filelist if globs_re.match(f)]
+            if m:
+                matches.update(m)
+            else:
+                logger.debug("removekmod %s: no files matched!", g)
+        remove_files = filelist.difference(matches)
+
+        if remove_files:
+            logger.debug("removekmod: removing %d files", len(remove_files))
+            list(remove(f) for f in remove_files)
+        else:
+            logger.debug("removekmod %s: no files to remove!", cmd)
+

+
[docs]    def createaddrsize(self, addr, src, dest):
+        '''
+        createaddrsize INITRD_ADDRESS INITRD ADDRSIZE
+          Create the initrd.addrsize file required in LPAR boot process.
+          Examples:
+            createaddrsize ${INITRD_ADDRESS} ${outroot}/${BOOTDIR}/initrd.img ${outroot}/${BOOTDIR}/initrd.addrsize
+        '''
+        addrsize = open(dest, "wb")
+        addrsize_data = struct.pack(">iiii", 0, int(addr, 16), 0, os.stat(src).st_size)
+        addrsize.write(addrsize_data)
+        addrsize.close()
+

+
[docs]    def systemctl(self, cmd, *units):
+        '''
+        systemctl [enable|disable|mask] UNIT [UNIT...]
+          Enable, disable, or mask the given systemd units.
+          Examples:
+            systemctl disable lvm2-monitor.service
+            systemctl mask fedora-storage-init.service fedora-configure.service
+        '''
+        if cmd not in ('enable', 'disable', 'mask'):
+            raise ValueError('unsupported systemctl cmd: %s' % cmd)
+        if not units:
+            logger.debug("systemctl: no units given for %s, ignoring", cmd)
+            return
+        self.mkdir("/run/systemd/system") # XXX workaround for systemctl bug
+        systemctl = ('systemctl', '--root', self.outroot, '--no-reload',
+                     '--quiet', cmd)
+        # XXX for some reason 'systemctl enable/disable' always returns 1
+        try:
+            cmd = systemctl + units
+            runcmd(cmd)
+        except CalledProcessError:
+            pass

+
+ +
+ +
+
+ +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/html/_modules/pylorax/sysutils.html b/docs/html/_modules/pylorax/sysutils.html new file mode 100644 index 00000000..756b87b0 --- /dev/null +++ b/docs/html/_modules/pylorax/sysutils.html @@ -0,0 +1,295 @@ + + + + + + + + + + pylorax.sysutils — Lorax 23.8 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + +
+ + + + + + +
+
+
+ +
+
+
+ +

Source code for pylorax.sysutils

+#
+# sysutils.py
+#
+# Copyright (C) 2009-2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Red Hat Author(s):  Martin Gracik <mgracik@redhat.com>
+#
+
+__all__ = ["joinpaths", "touch", "replace", "chown_", "chmod_", "remove",
+           "linktree"]
+
+import sys
+import os
+import re
+import fileinput
+import pwd
+import grp
+import glob
+import shutil
+
+from pylorax.executils import runcmd
+
+
[docs]def joinpaths(*args, **kwargs):
+    path = os.path.sep.join(args)
+
+    if kwargs.get("follow_symlinks"):
+        return os.path.realpath(path)
+    else:
+        return path
+
+

+
[docs]def touch(fname):
+    open(fname, "w")
+
+

+
[docs]def replace(fname, find, sub):
+    fin = fileinput.input(fname, inplace=1)
+    pattern = re.compile(find)
+
+    for line in fin:
+        line = pattern.sub(sub, line)
+        sys.stdout.write(line)
+
+    fin.close()
+
+

+
[docs]def chown_(path, user=None, group=None, recursive=False):
+    uid = gid = -1
+
+    if user is not None:
+        uid = pwd.getpwnam(user)[2]
+    if group is not None:
+        gid = grp.getgrnam(group)[2]
+
+    for fname in glob.iglob(path):
+        os.chown(fname, uid, gid)
+
+        if recursive and os.path.isdir(fname):
+            for nested in os.listdir(fname):
+                nested = joinpaths(fname, nested)
+                chown_(nested, user, group, recursive)
+
+

+
[docs]def chmod_(path, mode, recursive=False):
+    for fname in glob.iglob(path):
+        os.chmod(fname, mode)
+
+        if recursive and os.path.isdir(fname):
+            for nested in os.listdir(fname):
+                nested = joinpaths(fname, nested)
+                chmod_(nested, mode, recursive)
+
+

+def cpfile(src, dst):
+    shutil.copy2(src, dst)
+    if os.path.isdir(dst):
+        dst = joinpaths(dst, os.path.basename(src))
+
+    return dst
+
+def mvfile(src, dst):
+    if os.path.isdir(dst):
+        dst = joinpaths(dst, os.path.basename(src))
+    os.rename(src, dst)
+    return dst
+
+
[docs]def remove(target):
+    if os.path.isdir(target) and not os.path.islink(target):
+        shutil.rmtree(target)
+    else:
+        os.unlink(target)
+

+
[docs]def linktree(src, dst):
+    runcmd(["/bin/cp", "-alx", src, dst])

+
+ +
+ +
+
+ +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/html/_modules/pylorax/treebuilder.html b/docs/html/_modules/pylorax/treebuilder.html new file mode 100644 index 00000000..9560754f --- /dev/null +++ b/docs/html/_modules/pylorax/treebuilder.html @@ -0,0 +1,527 @@ + + + + + + + + + + pylorax.treebuilder — Lorax 23.8 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + +
+ + + + + + +
+
+
+ +
+
+
+ +

Source code for pylorax.treebuilder

+# treebuilder.py - handle arch-specific tree building stuff using templates
+#
+# Copyright (C) 2011-2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Author(s):  Will Woods <wwoods@redhat.com>
+
+import logging
+logger = logging.getLogger("pylorax.treebuilder")
+
+import os, re
+from os.path import basename
+from shutil import copytree, copy2
+
+from pylorax.sysutils import joinpaths, remove
+from pylorax.base import DataHolder
+from pylorax.ltmpl import LoraxTemplateRunner
+import pylorax.imgutils as imgutils
+from pylorax.executils import runcmd, runcmd_output
+
+templatemap = {
+    'i386':    'x86.tmpl',
+    'x86_64':  'x86.tmpl',
+    'ppc':     'ppc.tmpl',
+    'ppc64':   'ppc.tmpl',
+    'ppc64le': 'ppc64le.tmpl',
+    's390':    's390.tmpl',
+    's390x':   's390.tmpl',
+    'aarch64': 'aarch64.tmpl',
+    'arm':     'arm.tmpl',
+    'armhfp':  'arm.tmpl',
+}
+
+
[docs]def generate_module_info(moddir, outfile=None):
+    def module_desc(mod):
+        output = runcmd_output(["modinfo", "-F", "description", mod])
+        return output.strip()
+    def read_module_set(name):
+        return set(l.strip() for l in open(joinpaths(moddir,name)) if ".ko" in l)
+    modsets = {'scsi':read_module_set("modules.block"),
+               'eth':read_module_set("modules.networking")}
+
+    modinfo = list()
+    for root, _dirs, files in os.walk(moddir):
+        for modtype, modset in modsets.items():
+            for mod in modset.intersection(files):  # modules in this dir
+                (name, _ext) = os.path.splitext(mod) # foo.ko -> (foo, .ko)
+                desc = module_desc(joinpaths(root,mod)) or "%s driver" % name
+                modinfo.append(dict(name=name, type=modtype, desc=desc))
+
+    out = open(outfile or joinpaths(moddir,"module-info"), "w")
+    out.write("Version 0\n")
+    for mod in sorted(modinfo, key=lambda m: m.get('name')):
+        out.write('{name}\n\t{type}\n\t"{desc:.65}"\n'.format(**mod))
+

+
[docs]class RuntimeBuilder(object):
+    '''Builds the anaconda runtime image.'''
+    def __init__(self, product, arch, dbo, templatedir=None,
+                 installpkgs=None,
+                 add_templates=None,
+                 add_template_vars=None):
+        root = dbo.conf.installroot
+        # use a copy of product so we can modify it locally
+        product = product.copy()
+        product.name = product.name.lower()
+        self.vars = DataHolder(arch=arch, product=product, dbo=dbo, root=root,
+                               basearch=arch.basearch, libdir=arch.libdir)
+        self.dbo = dbo
+        self._runner = LoraxTemplateRunner(inroot=root, outroot=root,
+                                           dbo=dbo, templatedir=templatedir)
+        self.add_templates = add_templates or []
+        self.add_template_vars = add_template_vars or {}
+        self._installpkgs = installpkgs or []
+        self._runner.defaults = self.vars
+        self.dbo.reset()
+
+    def _install_branding(self):
+        release = None
+        q = self.dbo.sack.query()
+        a = q.available()
+        for pkg in a.filter(provides='/etc/system-release'):
+            if pkg.name.startswith('generic'):
+                continue
+            else:
+                release = pkg.name
+                break
+
+        if not release:
+            logger.error('could not get the release')
+            return
+
+        # release
+        logger.info('got release: %s', release)
+        self._runner.installpkg(release)
+
+        # logos
+        release, _suffix = release.split('-', 1)
+        self._runner.installpkg('%s-logos' % release)
+
+
[docs]    def install(self):
+        '''Install packages and do initial setup with runtime-install.tmpl'''
+        self._install_branding()
+        if len(self._installpkgs) > 0:
+            self._runner.installpkg(*self._installpkgs)
+        self._runner.run("runtime-install.tmpl")
+        for tmpl in self.add_templates:
+            self._runner.run(tmpl, **self.add_template_vars)
+

+
[docs]    def writepkglists(self, pkglistdir):
+        '''debugging data: write out lists of package contents'''
+        if not os.path.isdir(pkglistdir):
+            os.makedirs(pkglistdir)
+        q = self.dbo.sack.query()
+        for pkgobj in q.installed():
+            with open(joinpaths(pkglistdir, pkgobj.name), "w") as fobj:
+                for fname in pkgobj.files:
+                    fobj.write("{0}\n".format(fname))
+

+
[docs]    def postinstall(self):
+        '''Do some post-install setup work with runtime-postinstall.tmpl'''
+        # copy configdir into runtime root beforehand
+        configdir = joinpaths(self._runner.templatedir,"config_files")
+        configdir_path = "tmp/config_files"
+        fullpath = joinpaths(self.vars.root, configdir_path)
+        if os.path.exists(fullpath):
+            remove(fullpath)
+        copytree(configdir, fullpath)
+        self._runner.run("runtime-postinstall.tmpl", configdir=configdir_path)
+

+
[docs]    def cleanup(self):
+        '''Remove unneeded packages and files with runtime-cleanup.tmpl'''
+        self._runner.run("runtime-cleanup.tmpl")
+

+
[docs]    def writepkgsizes(self, pkgsizefile):
+        '''debugging data: write a big list of pkg sizes'''
+        fobj = open(pkgsizefile, "w")
+        getsize = lambda f: os.lstat(f).st_size if os.path.exists(f) else 0
+        q = self.dbo.sack.query()
+        for p in sorted(q.installed()):
+            pkgsize = sum(getsize(joinpaths(self.vars.root,f)) for f in p.files)
+            fobj.write("{0.name}.{0.arch}: {1}\n".format(p, pkgsize))
+

+
[docs]    def generate_module_data(self):
+        root = self.vars.root
+        moddir = joinpaths(root, "lib/modules/")
+        for kver in os.listdir(moddir):
+            ksyms = joinpaths(root, "boot/System.map-%s" % kver)
+            logger.info("doing depmod and module-info for %s", kver)
+            runcmd(["depmod", "-a", "-F", ksyms, "-b", root, kver])
+            generate_module_info(moddir+kver, outfile=moddir+"module-info")
+

+
[docs]    def create_runtime(self, outfile="/var/tmp/squashfs.img", compression="xz", compressargs=None, size=2):
+        # make live rootfs image - must be named "LiveOS/rootfs.img" for dracut
+        compressargs = compressargs or []
+        workdir = joinpaths(os.path.dirname(outfile), "runtime-workdir")
+        os.makedirs(joinpaths(workdir, "LiveOS"))
+
+        imgutils.mkrootfsimg(self.vars.root, joinpaths(workdir, "LiveOS/rootfs.img"),
+                             "Anaconda", size=size)
+
+        # squash the live rootfs and clean up workdir
+        imgutils.mksquashfs(workdir, outfile, compression, compressargs)
+        remove(workdir)
+

+
[docs]    def finished(self):
+        """ Done using RuntimeBuilder
+
+        Close the dnf base object
+        """
+        self.dbo.close()
+

+
[docs]class TreeBuilder(object):
+    '''Builds the arch-specific boot images.
+    inroot should be the installtree root (the newly-built runtime dir)'''
+    def __init__(self, product, arch, inroot, outroot, runtime, isolabel, domacboot=True, doupgrade=True, templatedir=None, add_templates=None, add_template_vars=None, workdir=None):
+
+        # NOTE: if you pass an arg named "runtime" to a mako template it'll
+        # clobber some mako internal variables - hence "runtime_img".
+        self.vars = DataHolder(arch=arch, product=product, runtime_img=runtime,
+                               runtime_base=basename(runtime),
+                               inroot=inroot, outroot=outroot,
+                               basearch=arch.basearch, libdir=arch.libdir,
+                               isolabel=isolabel, udev=udev_escape, domacboot=domacboot, doupgrade=doupgrade,
+                               workdir=workdir, lower=string_lower)
+        self._runner = LoraxTemplateRunner(inroot, outroot, templatedir=templatedir)
+        self._runner.defaults = self.vars
+        self.add_templates = add_templates or []
+        self.add_template_vars = add_template_vars or {}
+        self.templatedir = templatedir
+        self.treeinfo_data = None
+
+    @property
+
[docs]    def kernels(self):
+        return findkernels(root=self.vars.inroot)
+

+
[docs]    def rebuild_initrds(self, add_args=None, backup="", prefix=""):
+        '''Rebuild all the initrds in the tree. If backup is specified, each
+        initrd will be renamed with backup as a suffix before rebuilding.
+        If backup is empty, the existing initrd files will be overwritten.
+        If suffix is specified, the existing initrd is untouched and a new
+        image is built with the filename "${prefix}-${kernel.version}.img"
+        '''
+        add_args = add_args or []
+        dracut = ["dracut", "--nomdadmconf", "--nolvmconf"] + add_args
+        if not backup:
+            dracut.append("--force")
+
+        kernels = [kernel for kernel in self.kernels if hasattr(kernel, "initrd")]
+        if not kernels:
+            raise Exception("No initrds found, cannot rebuild_initrds")
+
+        # Hush some dracut warnings. TODO: bind-mount proc in place?
+        open(joinpaths(self.vars.inroot,"/proc/modules"),"w")
+        for kernel in kernels:
+            if prefix:
+                idir = os.path.dirname(kernel.initrd.path)
+                outfile = joinpaths(idir, prefix+'-'+kernel.version+'.img')
+            else:
+                outfile = kernel.initrd.path
+            logger.info("rebuilding %s", outfile)
+            if backup:
+                initrd = joinpaths(self.vars.inroot, outfile)
+                os.rename(initrd, initrd + backup)
+            cmd = dracut + [outfile, kernel.version]
+            runcmd(cmd, root=self.vars.inroot)
+
+            # ppc64 cannot boot images > 32MiB, check size and warn
+            if self.vars.arch.basearch in ("ppc64", "ppc64le") and os.path.exists(outfile):
+                st = os.stat(outfile)
+                if st.st_size > 32 * 1024 * 1024:
+                    logging.warning("ppc64 initrd %s is > 32MiB", outfile)
+
+        os.unlink(joinpaths(self.vars.inroot,"/proc/modules"))
+

+
[docs]    def build(self):
+        templatefile = templatemap[self.vars.arch.basearch]
+        for tmpl in self.add_templates:
+            self._runner.run(tmpl, **self.add_template_vars)
+        self._runner.run(templatefile, kernels=self.kernels)
+        self.treeinfo_data = self._runner.results.treeinfo
+        self.implantisomd5()
+

+
[docs]    def implantisomd5(self):
+        for _section, data in self.treeinfo_data.items():
+            if 'boot.iso' in data:
+                iso = joinpaths(self.vars.outroot, data['boot.iso'])
+                runcmd(["implantisomd5", iso])
+

+    @property
+
[docs]    def dracut_hooks_path(self):
+        """ Return the path to the lorax dracut hooks scripts
+
+            Use the configured share dir if it is setup,
+            otherwise default to /usr/share/lorax/dracut_hooks
+        """
+        if self.templatedir:
+            return joinpaths(self.templatedir, "dracut_hooks")
+        else:
+            return "/usr/share/lorax/dracut_hooks"
+

+
[docs]    def copy_dracut_hooks(self, hooks):
+        """ Copy the hook scripts in hooks into the installroot's /tmp/
+        and return a list of commands to pass to dracut when creating the
+        initramfs
+
+        hooks is a list of tuples with the name of the hook script and the
+        target dracut hook directory
+        (eg. [("99anaconda-copy-ks.sh", "/lib/dracut/hooks/pre-pivot")])
+        """
+        dracut_commands = []
+        for hook_script, dracut_path in hooks:
+            src = joinpaths(self.dracut_hooks_path, hook_script)
+            if not os.path.exists(src):
+                logger.error("Missing lorax dracut hook script %s", (src))
+                continue
+            dst = joinpaths(self.vars.inroot, "/tmp/", hook_script)
+            copy2(src, dst)
+            dracut_commands += ["--include", joinpaths("/tmp/", hook_script),
+                                dracut_path]
+        return dracut_commands
+
+#### TreeBuilder helper functions
+

+
[docs]def findkernels(root="/", kdir="boot"):
+    # To find possible flavors, awk '/BuildKernel/ { print $4 }' kernel.spec
+    flavors = ('debug', 'PAE', 'PAEdebug', 'smp', 'xen', 'lpae')
+    kre = re.compile(r"vmlinuz-(?P<version>.+?\.(?P<arch>[a-z0-9_]+)"
+                     r"(.(?P<flavor>{0}))?)$".format("|".join(flavors)))
+    kernels = []
+    bootfiles = os.listdir(joinpaths(root, kdir))
+    for f in bootfiles:
+        match = kre.match(f)
+        if match:
+            kernel = DataHolder(path=joinpaths(kdir, f))
+            kernel.update(match.groupdict()) # sets version, arch, flavor
+            kernels.append(kernel)
+
+    # look for associated initrd/initramfs/etc.
+    for kernel in kernels:
+        for f in bootfiles:
+            if f.endswith('-'+kernel.version+'.img'):
+                imgtype, _rest = f.split('-',1)
+                # special backwards-compat case
+                if imgtype == 'initramfs':
+                    imgtype = 'initrd'
+                kernel[imgtype] = DataHolder(path=joinpaths(kdir, f))
+
+    logger.debug("kernels=%s", kernels)
+    return kernels
+
+# udev whitelist: 'a-zA-Z0-9#+.:=@_-' (see is_whitelisted in libudev-util.c)

+udev_blacklist=' !"$%&\'()*,/;<>?[\\]^`{|}~' # ASCII printable, minus whitelist
+udev_blacklist += ''.join(chr(i) for i in range(32)) # ASCII non-printable
+
[docs]def udev_escape(label):
+    out = ''
+    for ch in label:
+        out += ch if ch not in udev_blacklist else '\\x%02x' % ord(ch)
+    return out
+

+
[docs]def string_lower(string):
+    """ Return a lowercase string.
+
+    :param string: String to lowercase
+
+    This is used as a filter in the templates.
+    """
+    return string.lower()

+
+ +
+ +
+
+ +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/html/_modules/pylorax/treeinfo.html b/docs/html/_modules/pylorax/treeinfo.html new file mode 100644 index 00000000..8ef20486 --- /dev/null +++ b/docs/html/_modules/pylorax/treeinfo.html @@ -0,0 +1,245 @@ + + + + + + + + + + pylorax.treeinfo — Lorax 23.8 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + +
+ + + + + + +
+
+
+ +
+
+
+ +

Source code for pylorax.treeinfo

+#
+# treeinfo.py
+#
+# Copyright (C) 2010-2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Red Hat Author(s):  Martin Gracik <mgracik@redhat.com>
+#
+
+import logging
+logger = logging.getLogger("pylorax.treeinfo")
+
+import configparser
+import time
+
+
+
[docs]class TreeInfo(object):
+
+    def __init__(self, product, version, variant, basearch,
+                 packagedir=""):
+
+        self.c = configparser.ConfigParser()
+
+        section = "general"
+        data = {"timestamp": str(time.time()),
+                "family": product,
+                "version": version,
+                "name": "%s-%s" % (product, version),
+                "variant": variant or "",
+                "arch": basearch,
+                "packagedir": packagedir}
+
+        self.c.add_section(section)
+        list(self.c.set(section, key, value) for key, value in data.items())
+
+
[docs]    def add_section(self, section, data):
+        if not self.c.has_section(section):
+            self.c.add_section(section)
+
+        list(self.c.set(section, key, value) for key, value in data.items())
+

+
[docs]    def write(self, outfile):
+        logger.info("writing .treeinfo file")
+        with open(outfile, "w") as fobj:
+            self.c.write(fobj)

+
+ +
+ +
+
+ +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/html/_sources/index.txt b/docs/html/_sources/index.txt new file mode 100644 index 00000000..c9b962d8 --- /dev/null +++ b/docs/html/_sources/index.txt @@ -0,0 +1,27 @@ +.. Lorax documentation master file, created by + sphinx-quickstart on Wed Apr 8 13:46:00 2015. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to Lorax's documentation! +================================= + +Contents: + +.. toctree:: + :maxdepth: 1 + + intro + lorax + livemedia-creator + product-images + modules + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/docs/html/_sources/intro.txt b/docs/html/_sources/intro.txt new file mode 100644 index 00000000..01857ee9 --- /dev/null +++ b/docs/html/_sources/intro.txt @@ -0,0 +1,67 @@ +Introduction to Lorax +===================== + +I am the Lorax. I speak for the trees [and images]. + +Lorax is used to build the Anaconda Installer boot.iso, it consists of a +library, pylorax, a set of templates, and the lorax script. Its operation +is driven by a customized set of Mako templates that lists the packages +to be installed, steps to execute to remove unneeded files, and creation +of the iso for all of the supported architectures. + + + + + + +Before Lorax +============ + +Tree building tools such as pungi and revisor rely on 'buildinstall' in +anaconda/scripts/ to produce the boot images and other such control files +in the final tree. The existing buildinstall scripts written in a mix of +bash and Python are unmaintainable. Lorax is an attempt to replace them +with something more flexible. + + +EXISTING WORKFLOW: + +pungi and other tools call scripts/buildinstall, which in turn call other +scripts to do the image building and data generation. Here's how it +currently looks: + + -> buildinstall + * process command line options + * write temporary yum.conf to point to correct repo + * find anaconda release RPM + * unpack RPM, pull in those versions of upd-instroot, mk-images, + maketreeinfo.py, makestamp.py, and buildinstall + + -> call upd-instroot + + -> call maketreeinfo.py + + -> call mk-images (which figures out which mk-images.ARCH to call) + + -> call makestamp.py + + * clean up + + +PROBLEMS: + +The existing workflow presents some problems with maintaining the scripts. +First, almost all knowledge of what goes in to the stage 1 and stage 2 +images lives in upd-instroot. The mk-images* scripts copy things from the +root created by upd-instroot in order to build the stage 1 image, though +it's not completely clear from reading the scripts. + + +NEW IDEAS: + +Create a new central driver with all information living in Python modules. +Configuration files will provide the knowledge previously contained in the +upd-instroot and mk-images* scripts. + + + diff --git a/docs/html/_sources/livemedia-creator.txt b/docs/html/_sources/livemedia-creator.txt new file mode 100644 index 00000000..3b8c3a2b --- /dev/null +++ b/docs/html/_sources/livemedia-creator.txt @@ -0,0 +1,500 @@ +livemedia-creator +================= + +:Authors: + Brian C. Lane + +livemedia-creator uses `Anaconda `_, +`kickstart `_ and `Lorax +`_ to create bootable media that use the +same install path as a normal system installation. It can be used to make live +isos, bootable (partitioned) disk images, tarfiles, and filesystem images for +use with virtualization and container solutions like libvirt, docker, and +OpenStack. + +The general idea is to use virt-install with kickstart and an Anaconda boot.iso +to install into a disk image and then use the disk image to create the bootable +media. + +livemedia-creator --help will describe all of the options available. At the +minimum you need: + +``--make-iso`` to create a final bootable .iso or one of the other ``--make-*`` options. + +``--iso`` to specify the Anaconda install media to use with virt-install + +``--ks`` to select the kickstart file describing what to install. + +To use livemedia-creator with virt-install you will need to install the +following packages, as well as have libvirtd setup correctly. + +* ``virt-install`` +* ``libvirt-python`` + +If you are going to be using Anaconda directly, with ``--no-virt`` mode, make sure +you have the anaconda package installed. You can use the anaconda-tui package +to save a bit of space on the build system. + +Conventions used in this document: + +``lmc`` is an abbreviation for livemedia-creator. + +``builder`` is the system where livemedia-creator is being run + +``image`` is the disk image being created by running livemedia-creator + + +Quickstart +---------- + +Run this to create a bootable live iso:: + + sudo livemedia-creator --make-iso \ + --iso=/extra/iso/boot.iso --ks=./docs/fedora-livemedia.ks + +You can run it directly from the lorax git repo like this:: + + sudo PATH=./src/sbin/:$PATH PYTHONPATH=./src/ ./src/sbin/livemedia-creator \ + --make-iso --iso=/extra/iso/boot.iso \ + --ks=./docs/fedora-livemedia.ks --lorax-templates=./share/ + +If you want to watch the install you can pass ``--vnc vnc`` and use a vnc client +to connect to localhost:0 + +This is usually a good idea when testing changes to the kickstart. lmc tries +to monitor the logs for fatal errors, but may not catch everything. + + +How ISO creation works +---------------------- + +There are 2 stages, the install stage which produces a disk or filesystem image +as its output, and the boot media creation which uses the image as its input. +Normally you would run both stages, but it is possible to stop after the +install stage, by using ``--image-only``, or to skip the install stage and use +a previously created disk image by passing ``--disk-image`` or ``--fs-image`` + +When creating an iso virt-install boots using the passed Anaconda installer iso +and installs the system based on the kickstart. The ``%post`` section of the +kickstart is used to customize the installed system in the same way that +current spin-kickstarts do. + +livemedia-creator monitors the install process for problems by watching the +install logs. They are written to the current directory or to the base +directory specified by the --logfile command. You can also monitor the install +by passing ``--vnc vnc`` and using a vnc client. This is recommended when first +modifying a kickstart, since there are still places where Anaconda may get +stuck without the log monitor catching it. + +The output from this process is a partitioned disk image. kpartx can be used +to mount and examine it when there is a problem with the install. It can also +be booted using kvm. + +When creating an iso the disk image's / partition is copied into a formatted +disk image which is then used as the input to lorax for creation of the final +media. + +The final image is created by lorax, using the templates in /usr/share/lorax/ +or the directory specified by ``--lorax-templates`` + +Currently the standard lorax templates are used to make a bootable iso, but +it should be possible to modify them to output other results. They are +written using the Mako template system which is very flexible. + + +Kickstarts +---------- + +The docs/ directory includes several example kickstarts, one to create a live +desktop iso using GNOME, and another to create a minimal disk image. When +creating your own kickstarts you should start with the minimal example, it +includes several needed packages that are not always included by dependencies. + +Or you can use existing spin kickstarts to create live media with a few +changes. Here are the steps I used to convert the Fedora XFCE spin. + +1. Flatten the xfce kickstart using ksflatten +2. Add zerombr so you don't get the disk init dialog +3. Add clearpart --all +4. Add swap partition +5. bootloader target +6. Add shutdown to the kickstart +7. Add network --bootproto=dhcp --activate to activate the network + This works for F16 builds but for F15 and before you need to pass + something on the cmdline that activate the network, like sshd: + + ``livemedia-creator --kernel-args="sshd"`` + +8. Add a root password:: + + rootpw rootme + network --bootproto=dhcp --activate + zerombr + clearpart --all + bootloader --location=mbr + part swap --size=512 + shutdown + +9. In the livesys script section of the %post remove the root password. This + really depends on how the spin wants to work. You could add the live user + that you create to the %wheel group so that sudo works if you wanted to. + + ``passwd -d root > /dev/null`` + +10. Remove /etc/fstab in %post, dracut handles mounting the rootfs + + ``cat /dev/null > /dev/fstab`` + + Do this only for live iso's, the filesystem will be mounted read only if + there is no /etc/fstab + +11. Don't delete initramfs files from /boot in %post +12. Have dracut-config-generic, grub-efi, memtest86+ and syslinux in the package + list. +13. Omit dracut-config-rescue from the %package list: ``-dracut-config-rescue`` + +One drawback to using virt-install is that it pulls the packages from +the repo each time you run it. To speed things up you either need a local +mirror of the packages, or you can use a caching proxy. When using a proxy +you pass it to livemedia-creator like this: + + ``--proxy=http://proxy.yourdomain.com:3128`` + +You also need to use a specific mirror instead of mirrormanager so that the +packages will get cached, so your kickstart url would look like: + + ``url --url="http://dl.fedoraproject.org/pub/fedora/linux/development/17/x86_64/os/"`` + +You can also add an update repo, but don't name it updates. Add --proxy to +it as well. + + +Anaconda image install (no-virt) +-------------------------------- + +You can create images without using virt-install by passing ``--no-virt`` on the +cmdline. This will use Anaconda's directory install feature to handle the install. +There are a couple of things to keep in mind when doing this: + +1. It will be most reliable when building images for the same release that the + host is running. Because Anaconda has expectations about the system it is + running under you may encounter strange bugs if you try to build newer or + older releases. + +2. Make sure selinux is set to permissive or disabled. It won't install + correctly with selinux set to enforcing yet. + +3. It may totally trash your host. So far I haven't had this happen, but the + possibility exists that a bug in Anaconda could result in it operating on + real devices. I recommend running it in a virt or on a system that you can + afford to lose all data from. + +The logs from anaconda will be placed in an ./anaconda/ directory in either +the current directory or in the directory used for --logfile + +Example cmdline: + +``sudo livemedia-creator --make-iso --no-virt --ks=./fedora-livemedia.ks`` + + +AMI Images +---------- + +Amazon EC2 images can be created by using the --make-ami switch and an appropriate +kickstart file. All of the work to customize the image is handled by the kickstart. +The example currently included was modified from the cloud-kickstarts version so +that it would work with livemedia-creator. + +Example cmdline: + +``sudo livemedia-creator --make-ami --iso=/path/to/boot.iso --ks=./docs/fedora-livemedia-ec2.ks`` + +This will produce an ami-root.img file in the working directory. + +At this time I have not tested the image with EC2. Feedback would be welcome. + + +Appliance Creation +------------------ + +livemedia-creator can now replace appliance-tools by using the --make-appliance +switch. This will create the partitioned disk image and an XML file that can be +used with virt-image to setup a virtual system. + +The XML is generated using the Mako template from +/usr/share/lorax/appliance/libvirt.xml You can use a different template by +passing ``--app-template