From b0b696d66d55fa23bbd0dc6e1783c5f4341c9a2f Mon Sep 17 00:00:00 2001 From: Martin Gracik Date: Thu, 4 Jun 2009 15:36:56 +0200 Subject: [PATCH] Wrote a templating system for initrd creation. A lot of stuff got changed and rewritten. Using a different approach now, so no point of tracking changes to the older commits. --- etc/templates/initrd | 9 + src/bin/lorax | 50 ++- src/pylorax/__init__.py | 386 +++++++++--------- .../{instroot.py => _backup/scrubs.py} | 222 ++++++---- src/pylorax/{scratch => _backup}/utils.py | 13 +- src/pylorax/actions/__init__.py | 28 ++ src/pylorax/actions/fileactions.py | 179 ++++++++ src/pylorax/base.py | 43 ++ src/pylorax/config.py | 125 ++++++ src/pylorax/exceptions.py | 7 + src/pylorax/fileutils.py | 50 --- src/pylorax/images.py | 206 ++++++---- src/pylorax/initrd.py | 74 ++++ src/pylorax/utils/__init__.py | 0 src/pylorax/utils/fileutil.py | 104 +++++ src/pylorax/utils/libutil.py | 57 +++ src/pylorax/utils/rpmutil.py | 158 +++++++ 17 files changed, 1265 insertions(+), 446 deletions(-) create mode 100644 etc/templates/initrd rename src/pylorax/{instroot.py => _backup/scrubs.py} (78%) rename src/pylorax/{scratch => _backup}/utils.py (94%) create mode 100644 src/pylorax/actions/__init__.py create mode 100644 src/pylorax/actions/fileactions.py create mode 100644 src/pylorax/base.py create mode 100644 src/pylorax/config.py create mode 100644 src/pylorax/exceptions.py delete mode 100644 src/pylorax/fileutils.py create mode 100644 src/pylorax/initrd.py create mode 100644 src/pylorax/utils/__init__.py create mode 100644 src/pylorax/utils/fileutil.py create mode 100644 src/pylorax/utils/libutil.py create mode 100644 src/pylorax/utils/rpmutil.py diff --git a/etc/templates/initrd b/etc/templates/initrd new file mode 100644 index 00000000..5a6984f4 --- /dev/null +++ b/etc/templates/initrd @@ -0,0 +1,9 @@ +# initrd template file +# +# supported variables: @initrd@ +# @instroot@ + +:copy + # comment + @instroot@/bin/bash to @initrd@/bin + @instroot@/bin/mount to @initrd@/bin diff --git a/src/bin/lorax b/src/bin/lorax index 8465ac1e..64159033 100755 --- a/src/bin/lorax +++ b/src/bin/lorax @@ -1,26 +1,5 @@ #!/usr/bin/env python -# # lorax -# Install image and tree support data generation tool. -# -# Copyright (C) 2008, 2009 Red Hat, Inc. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# Author(s): David Cantrell -# Martin Gracik -# import sys import os @@ -30,7 +9,7 @@ import pylorax if __name__ == '__main__': - version = '%s 0.1' % (os.path.basename(sys.argv[0]),) + version = '%s 0.1' % os.path.basename(sys.argv[0]) usage = '%prog -p PRODUCT -v VERSION -r RELEASE -o OUTPUT REPOSITORY' parser = OptionParser(usage=usage) @@ -39,7 +18,8 @@ if __name__ == '__main__': if os.path.isdir(value): setattr(parser.values, option.dest, value) else: - parser.error('%s is not a directory' % (value,)) + parser.error('%s is not a directory.' % value) + # XXX "options" should not be required # required @@ -65,7 +45,8 @@ if __name__ == '__main__': metavar='URL', default='your distribution provided bug reporting tool.') group.add_option('-u', '--updates', help='Path to find updated RPMS.', metavar='PATHSPEC') - group.add_option('-m', '--mirrorlist', help='Mirror list repository, may be listed multiple times.', + group.add_option('-m', '--mirrorlist', + help='Mirror list repository, may be listed multiple times.', metavar='REPOSITORY', action='append', default=[]) group.add_option('-c', '--confdir', help='Path to config files (default: /etc/lorax).', metavar='PATHSPEC', action='callback', callback=check_dir, @@ -89,10 +70,27 @@ if __name__ == '__main__': sys.exit(0) if not opts.product or not opts.version or not opts.release or not opts.output: - parser.error('Missing a required argument.') + parser.error('Missing required argument.') if not args: parser.error('Missing repository to use for image generation.') - lorax = pylorax.Lorax(repos=args, options=opts) + config = pylorax.Config() + config.set(confdir=opts.confdir, + debug=opts.debug, + cleanup=opts.cleanup) + + # required + config.set(product=opts.product, + version=opts.version, + release=opts.release, + outdir=opts.output, + repos=args) + # optional + config.set(variant=opts.variant, + bugurl=opts.bugurl, + updates=opts.updates, + mirrorlist=opts.mirrorlist) + + lorax = pylorax.Lorax(config=config) lorax.run() diff --git a/src/pylorax/__init__.py b/src/pylorax/__init__.py index 2acded81..8b2061b9 100644 --- a/src/pylorax/__init__.py +++ b/src/pylorax/__init__.py @@ -1,25 +1,4 @@ -# -# pylorax -# Install image and tree support data generation tool -- Python module. -# -# Copyright (C) 2008, 2009 Red Hat, Inc. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# Author(s): David Cantrell -# Martin Gracik -# +# __init__.py import sys import os @@ -27,209 +6,211 @@ import shutil import tempfile import time import ConfigParser +import re -import yum -import rpmUtils +from config import Container +import utils.rpmutil as rpmutil -import instroot import images +from exceptions import LoraxError -class Lorax: - def __init__(self, repos, options): - self.repos = repos - # required - self.product = options.product - self.version = options.version - self.release = options.release - self.output = options.output +class Config(Container): + def __init__(self): + config = ('confdir', 'datadir', 'tempdir', 'debug', 'cleanup') - # optional - self.debug = options.debug - self.variant = options.variant - self.bugurl = options.bugurl - self.updates = options.updates - self.mirrorlist = options.mirrorlist - self.confdir = options.confdir - self.cleanup = options.cleanup + # options + required = ('product', 'version', 'release', 'outdir', 'repos') + optional = ('variant', 'bugurl', 'updates', 'mirrorlist') - self.conf = {} - if not self.confdir: - self.confdir = '/etc/lorax' - self.conf['confdir'] = self.confdir - self.conf['datadir'] = '/usr/share/lorax' - self.conf['tmpdir'] = tempfile.gettempdir() + Container.__init__(self, config + required + optional) + + # set defaults + self.set(confdir='/etc/lorax', + datadir='/usr/share/lorax', + tempdir=tempfile.mkdtemp(prefix='lorax.tmp.', dir=tempfile.gettempdir()), + debug=False, + cleanup=False) + + self.set(product='', + version='', + release='', + outdir='', + repos=[]) + + self.set(variant='', + bugurl='', + updates='', + mirrorlist=[]) + + +class Lorax(object): + def __init__(self, config): + assert isinstance(config, Config) == True + self.conf = config + + # check if we have all required options + if not self.conf.repos: + raise LoraxError, 'missing repos' + if not self.conf.outdir: + raise LoraxError, 'missing outdir' + if not self.conf.product: + raise LoraxError, 'missing product' + if not self.conf.version: + raise LoraxError, 'missing version' + if not self.conf.release: + raise LoraxError, 'missing release' + + self.yum = None def run(self): - """run() - - Generate install images. - """ - print('Collecting repos...') - self.repo, self.extrarepos = self.__collectRepos() + self.collectRepos() - if not self.repo: - sys.stderr.write('No valid repository.\n') + # check if we have at least one valid repository + if not self.conf.repo: + sys.stderr.write('ERROR: no valid repository\n') sys.exit(1) print('Initializing directories...') - self.buildinstdir, self.treedir, self.cachedir = self.__initializeDirs() + self.initDirs() - print('Writing yum configuration...') - self.yumconf = self.__writeYumConf() + print('Initializing yum...') + self.initYum() - print('Getting the build architecture...') - self.buildarch = self.__getBuildArch() - - print('Creating install root tree...') - self.makeInstRoot() + print('Setting build architecture...') + self.setBuildArch() print('Writing .treeinfo...') - self.__writeTreeInfo() + self.writeTreeInfo() print('Writing .discinfo...') - self.__writeDiscInfo() + self.writeDiscInfo() - print('Creating images...') + print('Preparing the install tree...') + self.prepareInstRoot() + + print('Creating the images...') self.makeImages() - if self.cleanup: + if self.conf.cleanup: print('Cleaning up...') self.cleanUp() - def __collectRepos(self): - """_collectRepos() - - Get the main repo (the first one) and then build a list of all remaining - repos in the list. Sanitize each repo URL for proper yum syntax. - """ - + def collectRepos(self): repolist = [] - for repospec in self.repos: + for repospec in self.conf.repos: if repospec.startswith('/'): - repo = 'file://%s' % (repospec,) - print('Adding local repo:\n %s' % (repo,)) + repo = 'file://%s' % repospec + print('Adding local repo: %s' % repo) repolist.append(repo) elif repospec.startswith('http://') or repospec.startswith('ftp://'): - print('Adding remote repo:\n %s' % (repospec,)) + print('Adding remote repo: %s' % repospec) repolist.append(repospec) if not repolist: - return None, [] + repo, extrarepos = None, [] else: - return repolist[0], repolist[1:] + repo, extrarepos = repolist[0], repolist[1:] - def __initializeDirs(self): - """_initializeDirs() + self.conf.addAttr(['repo', 'extrarepos']) + self.conf.set(repo=repo, extrarepos=extrarepos) - Create directories used for image generation. - """ + # remove repos attribute, to get a traceback, if we use it later + self.conf.delAttr('repos') - if not os.path.isdir(self.output): - os.makedirs(self.output, mode=0755) + def initDirs(self): + if not os.path.isdir(self.conf.outdir): + os.makedirs(self.conf.outdir, mode=0755) - self.conf['tmpdir'] = tempfile.mkdtemp('XXXXXX', 'lorax.tmp.', self.conf['tmpdir']) - buildinstdir = tempfile.mkdtemp('XXXXXX', 'buildinstall.tree.', self.conf['tmpdir']) - treedir = tempfile.mkdtemp('XXXXXX', 'treedir.', self.conf['tmpdir']) - cachedir = tempfile.mkdtemp('XXXXXX', 'yumcache.', self.conf['tmpdir']) + treedir = os.path.join(self.conf.tempdir, 'treedir', 'install') + cachedir = os.path.join(self.conf.tempdir, 'yumcache') print('Working directories:') - print(' tmpdir = %s' % (self.conf['tmpdir'],)) - print(' buildinstdir = %s' % (buildinstdir,)) - print(' treedir = %s' % (treedir,)) - print(' cachedir = %s' % (cachedir,)) + print(' tempdir = %s' % self.conf.tempdir) + print(' treedir = %s' % treedir) + print(' cachedir = %s' % cachedir) - return buildinstdir, treedir, cachedir - - def __writeYumConf(self): - """_writeYumConf() - - Generate a temporary yum.conf file for image generation. Returns the path - to the temporary yum.conf file on success, None of failure. - """ - - (fd, yumconf) = tempfile.mkstemp(prefix='yum.conf', dir=self.conf['tmpdir']) - f = os.fdopen(fd, 'w') - - f.write('[main]\n') - f.write('cachedir=%s\n' % (self.cachedir,)) - f.write('keepcache=0\n') - f.write('gpgcheck=0\n') - f.write('plugins=0\n') - f.write('reposdir=\n') - f.write('tsflags=nodocs\n\n') - - f.write('[loraxrepo]\n') - f.write('name=lorax repo\n') - f.write('baseurl=%s\n' % (self.repo,)) - f.write('enabled=1\n\n') - - for n, extra in enumerate(self.extrarepos, start=1): - f.write('[lorax-extrarepo-%d]\n' % (n,)) - f.write('name=lorax extra repo %d\n' % (n,)) - f.write('baseurl=%s\n' % (extra,)) - f.write('enabled=1\n') - - for n, mirror in enumerate(self.mirrorlist, start=1): - f.write('[lorax-mirrorlistrepo-%d]\n' % (n,)) - f.write('name=lorax mirrorlist repo %d\n' % (n,)) - f.write('mirrorlist=%s\n' % (mirror,)) - f.write('enabled=1\n') - - f.close() - print('Wrote lorax yum configuration to %s' % (yumconf,)) - - return yumconf - - def __getBuildArch(self): - """_getBuildArch() - - Query the configured yum repositories to determine our build architecture, - which is the architecture of the anaconda package in the repositories. - - This function is based on a subset of what repoquery(1) does. - """ - - uname_arch = os.uname()[4] - - if not self.yumconf or not os.path.isfile(self.yumconf): - sys.stderr.write('ERROR: yum.conf does not exist, defaulting to %s\n' % (uname_arch,)) - return uname_arch - - repoq = yum.YumBase() - repoq.doConfigSetup(self.yumconf) + self.conf.addAttr(['treedir', 'cachedir']) + self.conf.set(treedir=treedir, cachedir=cachedir) + def initYum(self): + yumconf = os.path.join(self.conf.tempdir, 'yum.conf') + try: - repoq.doRepoSetup() - except yum.Errors.RepoError: - sys.stderr.write('ERROR: cannot query yum repo, defaulting to %s\n' % (uname_arch,)) - return uname_arch + f = open(yumconf, 'w') + except IOError: + sys.stderr.write('ERROR: Unable to write yum.conf file\n') + sys.exit(1) + else: + f.write('[main]\n') + f.write('cachedir=%s\n' % self.conf.cachedir) + f.write('keepcache=0\n') + f.write('gpgcheck=0\n') + f.write('plugins=0\n') + f.write('reposdir=\n') + f.write('tsflags=nodocs\n\n') - repoq.doSackSetup(rpmUtils.arch.getArchList()) - repoq.doTsSetup() + f.write('[loraxrepo]\n') + f.write('name=lorax repo\n') + f.write('baseurl=%s\n' % self.conf.repo) + f.write('enabled=1\n\n') - ret_arch = None - for pkg in repoq.pkgSack.simplePkgList(): - (n, a, e, v, r) = pkg - if n == 'anaconda': - ret_arch = a - break + for n, extra in enumerate(self.conf.extrarepos, start=1): + f.write('[lorax-extrarepo-%d]\n' % n) + f.write('name=lorax extra repo %d\n' % n) + f.write('baseurl=%s\n' % extra) + f.write('enabled=1\n') - if not ret_arch: - ret_arch = uname_arch + for n, mirror in enumerate(self.conf.mirrorlist, start=1): + f.write('[lorax-mirrorlistrepo-%d]\n' % n) + f.write('name=lorax mirrorlist repo %d\n' % n) + f.write('mirrorlist=%s\n' % mirror) + f.write('enabled=1\n') - return ret_arch + f.close() - def __writeTreeInfo(self, discnum=1, totaldiscs=1, packagedir=''): - outfile = os.path.join(self.output, '.treeinfo') + self.conf.addAttr('yumconf') + self.conf.set(yumconf=yumconf) + self.yum = rpmutil.Yum(yumconf=self.conf.yumconf, installroot=self.conf.treedir) + + # remove not needed options + self.conf.delAttr(['repo', 'extrarepos', 'mirrorlist']) + + def setBuildArch(self): + unamearch = os.uname()[4] + + self.conf.addAttr('buildarch') + self.conf.set(buildarch=unamearch) + + anaconda = self.yum.find('anaconda') + try: + self.conf.set(buildarch=anaconda[0].arch) + except: + pass + + # set the libdir + self.conf.addAttr('libdir') + self.conf.set(libdir='lib') + # on 64-bit systems, make sure we use lib64 as the lib directory + if self.conf.buildarch.endswith('64') or self.conf.buildarch == 's390x': + self.conf.set(libdir='lib64') + + def writeTreeInfo(self, discnum=1, totaldiscs=1, packagedir=''): + outfile = os.path.join(self.conf.outdir, '.treeinfo') + + # don't print anything instead of None if variant is not specified + variant = '' + if self.conf.variant: + variant = self.conf.variant + data = { 'timestamp': time.time(), - 'family': self.product, - 'version': self.version, - 'arch': self.buildarch, - 'variant': self.variant, + 'family': self.conf.product, + 'version': self.conf.version, + 'arch': self.conf.buildarch, + 'variant': variant, 'discnum': str(discnum), 'totaldiscs': str(totaldiscs), 'packagedir': packagedir } @@ -241,6 +222,17 @@ class Lorax: for key, value in data.items(): c.set(section, key, value) + section = 'images-%s' % self.conf.buildarch + c.add_section(section) + c.set(section, 'kernel', 'images/pxeboot/vmlinuz') + c.set(section, 'initrd', 'images/pxeboot/initrd.img') + + # XXX actually create the boot iso somewhere, and set up this attribute + self.conf.addAttr('bootiso') + + if self.conf.bootiso: + c.set(section, 'boot.iso', 'images/%s' % self.conf.bootiso) + try: f = open(outfile, 'w') except IOError: @@ -250,47 +242,43 @@ class Lorax: f.close() return True - def __writeDiscInfo(self, discnum=0): - outfile = os.path.join(self.output, '.discinfo') + def writeDiscInfo(self, discnum=0): + outfile = os.path.join(self.conf.outdir, '.discinfo') try: f = open(outfile, 'w') except IOError: return False else: - f.write('%f\n' % (time.time(),)) - f.write('%s\n' % (self.release,)) - f.write('%s\n' % (self.buildarch,)) - f.write('%d\n' % (discnum,)) + f.write('%f\n' % time.time()) + f.write('%s\n' % self.conf.release) + f.write('%s\n' % self.conf.buildarch) + f.write('%d\n' % discnum) f.close() return True - def makeInstRoot(self): - root = instroot.InstRoot(conf=self.conf, - yumconf=self.yumconf, - arch=self.buildarch, - treedir=self.treedir, - updates=self.updates) - root.run() + def prepareInstRoot(self): + # XXX why do we need this? + os.symlink(os.path.join(os.path.sep, 'tmp'), + os.path.join(self.conf.treedir, 'var', 'lib', 'xkb')) - # TODO def makeImages(self): - pass + i = images.Images(self.conf, self.yum) + i.run() + + # XXX figure out where to put this + #def copyUpdates(self): + # if self.conf.updates and os.path.isdir(self.conf.updates): + # cp(os.path.join(self.conf.updates, '*'), self.conf.treedir) + # self.conf.delAttr('updates') def cleanUp(self, trash=[]): - """cleanup([trash]) - - Given a list of things to remove, cleanUp() will remove them if it can. - Never fails, just tries to remove things and returns regardless of - failures removing things. - """ - for item in trash: if os.path.isdir(item): shutil.rmtree(item, ignore_errors=True) else: os.unlink(item) - if os.path.isdir(self.conf['tmpdir']): - shutil.rmtree(self.conf['tmpdir'], ignore_errors=True) - + # remove the whole lorax tempdir + if os.path.isdir(self.conf.tempdir): + shutil.rmtree(self.conf.tempdir, ignore_errors=True) diff --git a/src/pylorax/instroot.py b/src/pylorax/_backup/scrubs.py similarity index 78% rename from src/pylorax/instroot.py rename to src/pylorax/_backup/scrubs.py index 00b00db3..74775f7f 100644 --- a/src/pylorax/instroot.py +++ b/src/pylorax/_backup/scrubs.py @@ -30,17 +30,18 @@ import fnmatch import re import fileinput import subprocess - import pwd import grp +import tempfile -from fileutils import cp, mv +from fileutils import cp, mv, touch +from yumutils import extract_rpm + +import utils -sys.path.insert(0, '/usr/share/yum-cli') -import yummain class InstRoot: - """InstRoot(conf, yumconf, arch, treedir, [updates=None]) + """InstRoot(config, options, yum) Create a instroot tree for the specified architecture. The list of packages to install are specified in the /etc/lorax/packages and @@ -58,20 +59,12 @@ class InstRoot: returned or the program is aborted immediately. """ - def __init__(self, conf, yumconf, arch, treedir, updates=None): - self.conf = conf - self.yumconf = yumconf - self.arch = arch - self.treedir = treedir - self.updates = updates + def __init__(self, config, options, yum): + self.conf = config + self.opts = options + self.yum = yum - self.libdir = 'lib' - # on 64-bit systems, make sure we use lib64 as the lib directory - if self.arch.endswith('64') or self.arch == 's390x': - self.libdir = 'lib64' - - # the directory where the instroot will be created - self.destdir = os.path.join(self.treedir, 'install') + self.destdir = self.conf.treedir def run(self): """run() @@ -79,24 +72,21 @@ class InstRoot: Generate the instroot tree and prepare it for building images. """ - if not os.path.isdir(self.destdir): - os.makedirs(self.destdir) - - # build a list of packages to install self.packages = self.__getPackageList() - - # install the packages to the instroot self.__installPackages() - # scrub instroot - self.__scrubInstRoot() + if not self.__installKernel(): + sys.exit(1) + + # XXX + #self.__scrubInstRoot() def __getPackageList(self): packages = set() packages_files = [] - packages_files.append(os.path.join(self.conf['confdir'], 'packages')) - packages_files.append(os.path.join(self.conf['confdir'], self.arch, 'packages')) + packages_files.append(os.path.join(self.conf.confdir, 'packages')) + packages_files.append(os.path.join(self.conf.confdir, self.opts.buildarch, 'packages')) for pfile in packages_files: if os.path.isfile(pfile): @@ -120,29 +110,98 @@ class InstRoot: return packages def __installPackages(self): - # build the list of arguments to pass to yum - arglist = ['-c', self.yumconf] - arglist.append("--installroot=%s" % (self.destdir,)) - arglist.extend(['install', '-y']) - arglist.extend(self.packages) - + # XXX i don't think this is needed # do some prep work on the destdir before calling yum - os.makedirs(os.path.join(self.destdir, 'boot')) - os.makedirs(os.path.join(self.destdir, 'usr', 'sbin')) - os.makedirs(os.path.join(self.destdir, 'usr', 'lib', 'debug')) - os.makedirs(os.path.join(self.destdir, 'usr', 'src', 'debug')) - os.makedirs(os.path.join(self.destdir, 'tmp')) - os.makedirs(os.path.join(self.destdir, 'var', 'log')) - os.makedirs(os.path.join(self.destdir, 'var', 'lib')) - os.makedirs(os.path.join(self.destdir, 'var', 'lib', 'yum')) + #os.makedirs(os.path.join(self.destdir, 'boot')) + #os.makedirs(os.path.join(self.destdir, 'usr', 'sbin')) + #os.makedirs(os.path.join(self.destdir, 'usr', 'lib', 'debug')) + #os.makedirs(os.path.join(self.destdir, 'usr', 'src', 'debug')) + #os.makedirs(os.path.join(self.destdir, 'tmp')) + #os.makedirs(os.path.join(self.destdir, 'var', 'log')) + #os.makedirs(os.path.join(self.destdir, 'var', 'lib', 'yum')) + + # XXX maybe only this... + #os.makedirs(os.path.join(self.destdir, 'var', 'lib')) os.symlink(os.path.join(os.path.sep, 'tmp'), os.path.join(self.destdir, 'var', 'lib', 'xkb')) - # XXX sort through yum errcodes and return False for actual bad things we care about - errcode = yummain.user_main(arglist, exit_code=False) + self.yum.install(self.packages) - # copy updates to destdir - if self.updates and os.path.isdir(self.updates): - cp(os.path.join(self.updates, '*'), self.destdir) + # copy updates to treedir + if self.opts.updates and os.path.isdir(self.opts.updates): + cp(os.path.join(self.opts.updates, '*'), self.destdir) + + # XXX + def __installKernel(self): + arches = [self.opts.buildarch] + efiarch = [] + kerneltags = ['kernel'] + kernelxen = [] + + if self.opts.buildarch == 'ppc': + arches = ['ppc64', 'ppc'] + elif self.opts.buildarch == 'i386': + arches = ['i586'] + efiarch = ['ia32'] + kerneltags = ['kernel', 'kernel-PAE'] + kernelxen = ['kernel-PAE'] + elif self.opts.buildarch == 'x86_64': + efiarch = ['x64'] + elif self.opts.buildarch == 'ia64': + efiarch = ['ia64'] + + kpackages = self.yum.find(kerneltags) + + if not kpackages: + sys.stderr.write('ERROR: Unable to find any kernel package\n') + return False + + # create the modinfo file + (fd, modinfo) = tempfile.mkstemp(prefix='modinfo-%s.' % self.opts.buildarch, + dir=self.conf.tempdir) + self.conf.addAttr('modinfo') + self.conf.set(modinfo=modinfo) + + for kernel in kpackages: + fn = self.yum.download(kernel) + kernelroot = os.path.join(self.conf.kernelbase, kernel.arch) + extract_rpm(fn, kernelroot) + os.unlink(fn) + + # get vmlinuz and version + dir = os.path.join(kernelroot, 'boot') + if self.opts.buildarch == 'ia64': + dir = os.path.join(dir, 'efi', 'EFI', 'redhat') + + vmlinuz = None + for file in os.listdir(dir): + if file.startswith('vmlinuz'): + vmlinuz = file + prefix, sep, version = file.partition('-') + + if not vmlinuz: + sys.stderr.write('ERROR: vmlinuz file not found\n') + return False + + modules_dir = os.path.join(kernelroot, 'lib', 'modules', version) + if not os.path.isdir(modules_dir): + sys.stderr.write('ERROR: modules directory not found\n') + return False + + allmods = [] + for file in os.listdir(modules_dir): + if file.endswith('.ko'): + allmods.append(os.path.join(modules_dir, file)) + + # install firmware + fpackages = self.yum.find('*firmware*') + for firmware in fpackages: + fn = self.yum.download(firmware) + extract_rpm(fn, kernelroot) + os.unlink(fn) + + utils.genmodinfo(modules_dir, self.conf.modinfo) + + return True def __scrubInstRoot(self): self.__createConfigFiles() @@ -155,35 +214,34 @@ class InstRoot: self.__configureKmod() self.__moveAnacondaFiles() self.__setShellLinks() - self.__moveBins() + #self.__moveBins() self.__removeUnwanted() self.__changeDestDirPermissions() self.__createLDConfig() self.__setBusyboxLinks() self.__strip() - self.__fixBrokenLinks() + #self.__fixBrokenLinks() def __createConfigFiles(self): # create %gconf.xml - dogtailconf = os.path.join(self.conf['datadir'], 'dogtail-%gconf.xml') + dogtailconf = os.path.join(self.conf.datadir, 'dogtail-%gconf.xml') if os.path.isfile(dogtailconf): os.makedirs(os.path.join(self.destdir, '.gconf', 'desktop', 'gnome', 'interface')) - f = open(os.path.join(self.destdir, '.gconf', 'desktop', '%gconf.xml'), 'w') - f.close() - f = open(os.path.join(self.destdir, '.gconf', 'desktop', 'gnome', '%gconf.xml'), 'w') - f.close() + touch(os.path.join(self.destdir, '.gconf', 'desktop', '%gconf.xml')) + touch(os.path.join(self.destdir, '.gconf', 'desktop', 'gnome', '%gconf.xml')) + dst = os.path.join(self.destdir, '.gconf', 'desktop', 'gnome', 'interface', '%gconf.xml') cp(dogtailconf, dst) # create selinux config if os.path.isfile(os.path.join(self.destdir, 'etc', 'selinux', 'targeted')): - selinuxconf = os.path.join(self.conf['datadir'], 'selinux-config') + selinuxconf = os.path.join(self.conf.datadir, 'selinux-config') if os.path.isfile(selinuxconf): dst = os.path.join(self.destdir, 'etc', 'selinux', 'config') cp(selinuxconf, dst) # create libuser.conf - libuserconf = os.path.join(self.conf['datadir'], 'libuser.conf') + libuserconf = os.path.join(self.conf.datadir, 'libuser.conf') if os.path.isfile(libuserconf): dst = os.path.join(self.destdir, 'etc', 'libuser.conf') cp(libuserconf, dst) @@ -265,7 +323,7 @@ class InstRoot: shutil.rmtree(icon, ignore_errors=True) # remove engines we don't need - tmp_path = os.path.join(self.destdir, 'usr', self.libdir, 'gtk-2.0') + tmp_path = os.path.join(self.destdir, 'usr', self.opts.libdir, 'gtk-2.0') if os.path.isdir(tmp_path): fnames = map(lambda fname: os.path.join(tmp_path, fname, 'engines'), os.listdir(tmp_path)) dnames = filter(lambda fname: os.path.isdir(fname), fnames) @@ -358,27 +416,27 @@ class InstRoot: if not os.path.isdir(bootpath): os.makedirs(bootpath) - if self.arch == 'i386' or self.arch == 'x86_64': + if self.opts.buildarch == 'i386' or self.opts.buildarch == 'x86_64': for bootfile in os.listdir(os.path.join(self.destdir, 'boot')): if bootfile.startswith('memtest'): src = os.path.join(self.destdir, 'boot', bootfile) dst = os.path.join(bootpath, bootfile) cp(src, dst) - elif self.arch.startswith('sparc'): + elif self.opts.buildarch.startswith('sparc'): for bootfile in os.listdir(os.path.join(self.destdir, 'boot')): if bootfile.endswith('.b'): src = os.path.join(self.destdir, 'boot', bootfile) dst = os.path.join(bootpath, bootfile) cp(src, dst) - elif self.arch.startswith('ppc'): + elif self.opts.buildarch.startswith('ppc'): src = os.path.join(self.destdir, 'boot', 'efika.forth') dst = os.path.join(bootpath, 'efika.forth') cp(src, dst) - elif self.arch == 'alpha': + elif self.opts.buildarch == 'alpha': src = os.path.join(self.destdir, 'boot', 'bootlx') dst = os.path.join(bootpath, 'bootlx') cp(src, dst) - elif self.arch == 'ia64': + elif self.opts.buildarch == 'ia64': src = os.path.join(self.destdir, 'boot', 'efi', 'EFI', 'redhat') shutil.rmtree(bootpath, ignore_errors=True) cp(src, bootpath) @@ -435,6 +493,7 @@ class InstRoot: os.symlink(busybox, sh) def __moveBins(self): + # XXX why do we want to move everything to /usr when in mk-images we copy it back? bin = os.path.join(self.destdir, 'bin') sbin = os.path.join(self.destdir, 'sbin') @@ -518,49 +577,47 @@ class InstRoot: def __createLDConfig(self): ldsoconf = os.path.join(self.destdir, 'etc', 'ld.so.conf') - f = open(ldsoconf, 'w') - f.close() + touch(ldsoconf) proc_dir = os.path.join(self.destdir, 'proc') if not os.path.isdir(proc_dir): os.makedirs(proc_dir) - # XXX isn't there a better way? - os.system('mount -t proc proc %s' % (proc_dir,)) + os.system('mount -t proc proc %s' % proc_dir) f = open(ldsoconf, 'w') - x11_libdir = os.path.join(self.destdir, 'usr', 'X11R6', self.libdir) + + x11_libdir = os.path.join(self.destdir, 'usr', 'X11R6', self.opts.libdir) if os.path.exists(x11_libdir): - f.write('/usr/X11R6/%s\n' % (self.libdir,)) - f.write('/usr/kerberos/%s\n' % (self.libdir,)) + f.write('/usr/X11R6/%s\n' % self.opts.libdir) + + f.write('/usr/kerberos/%s\n' % self.opts.libdir) cwd = os.getcwd() os.chdir(self.destdir) - # XXX can't exit from os.chroot() :( - os.system('/usr/sbin/chroot %s /usr/sbin/ldconfig' % (self.destdir,)) + os.system('/usr/sbin/chroot %s /sbin/ldconfig' % self.destdir) os.chdir(cwd) - if self.arch not in ('s390', 's390x'): - os.unlink(os.path.join(self.destdir, 'usr', 'sbin', 'ldconfig')) + if self.opts.buildarch not in ('s390', 's390x'): + os.unlink(os.path.join(self.destdir, 'sbin', 'ldconfig')) os.unlink(os.path.join(self.destdir, 'etc', 'ld.so.conf')) - # XXX isn't there a better way? - os.system('umount %s' % (proc_dir,)) + os.system('umount %s' % proc_dir) def __setBusyboxLinks(self): - src = os.path.join(self.destdir, 'usr', 'sbin', 'busybox.anaconda') - dst = os.path.join(self.destdir, 'usr', 'bin', 'busybox') + src = os.path.join(self.destdir, 'sbin', 'busybox.anaconda') + dst = os.path.join(self.destdir, 'bin', 'busybox') mv(src, dst) cwd = os.getcwd() - os.chdir(os.path.join(self.destdir, 'usr', 'bin')) + os.chdir(os.path.join(self.destdir, 'bin')) busybox_process = subprocess.Popen(['./busybox'], stdout=subprocess.PIPE) busybox_process.wait() if busybox_process.returncode: - raise LoraxError, 'cannot run busybox' + raise Error, 'busybox error' busybox_output = busybox_process.stdout.readlines() busybox_output = map(lambda line: line.strip(), busybox_output) @@ -641,3 +698,10 @@ class InstRoot: newtarget = re.sub(r'^\.\./\.\./%s/\(.*\)' % dir, r'\.\./%s/\1' % dir, target) if newtarget != target: os.symlink(newtarget, link) + + def __makeAdditionalDirs(self): + os.makedirs(os.path.join(self.destdir, 'modules')) + os.makedirs(os.path.join(self.destdir, 'tmp')) + for dir in ('a', 'b', 'd', 'l', 's', 'v', 'x'): + os.makedirs(os.path.join(self.destdir, 'etc', 'terminfo', dir)) + os.makedirs(os.path.join(self.destdir, 'var', 'lock', 'rpm')) diff --git a/src/pylorax/scratch/utils.py b/src/pylorax/_backup/utils.py similarity index 94% rename from src/pylorax/scratch/utils.py rename to src/pylorax/_backup/utils.py index 93291678..f6191427 100644 --- a/src/pylorax/scratch/utils.py +++ b/src/pylorax/_backup/utils.py @@ -9,11 +9,7 @@ logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger('pylorax') -def genmodinfo(path=None): - if not path: - release = os.uname()[2] - path = os.path.join('/lib/modules', release) - +def genmodinfo(path, output): mods = {} for root, dirs, files in os.walk(path): for file in files: @@ -56,13 +52,14 @@ def genmodinfo(path=None): modinfo = '%s\n\t%s\n\t"%s"\n' % (modname, modtype, desc) list[modtype][modname] = modinfo - # XXX i don't think we want to print it out now - print 'Version 0' + f = open(output, 'a') + f.write('Version 0\n') for type in list: modlist = list[type].keys() modlist.sort() for m in modlist: - print list[type][m] + f.write('%s\n' %list[type][m]) + f.close() # XXX is the input a path to a file? diff --git a/src/pylorax/actions/__init__.py b/src/pylorax/actions/__init__.py new file mode 100644 index 00000000..eba3a70c --- /dev/null +++ b/src/pylorax/actions/__init__.py @@ -0,0 +1,28 @@ +# pylorax/actions/__init__.py + +import os + + +def getActions(): + actions = {} + root, actions_dir = os.path.split(os.path.dirname(__file__)) + + modules = set() + for filename in os.listdir(os.path.join(root, actions_dir)): + if filename.endswith('.py') and filename != '__init__.py': + basename, extension = os.path.splitext(filename) + modules.add(os.path.join('pylorax', actions_dir, basename).replace('/', '.')) + + for module in modules: + print('Loading actions from %s' % module) + imported = __import__(module, globals(), locals(), [module], -1) + try: + commands = getattr(imported, 'COMMANDS') + except AttributeError: + continue + else: + for command, classname in commands.items(): + print('Loaded: %s' % classname) + actions[command] = getattr(imported, classname) + + return actions diff --git a/src/pylorax/actions/fileactions.py b/src/pylorax/actions/fileactions.py new file mode 100644 index 00000000..dcbb1c04 --- /dev/null +++ b/src/pylorax/actions/fileactions.py @@ -0,0 +1,179 @@ +# pylorax/actions/fileactions.py + +from pylorax.base import LoraxAction + +import os +import re +from pylorax.utils.fileutil import cp, mv, touch, edit, replace + + +COMMANDS = { 'copy': 'Copy', + 'move': 'Move', + 'link': 'Link', + 'touch': 'Touch', + 'edit': 'Edit', + 'replace': 'Replace' } + + +def getFileName(string): + m = re.match(r'@instroot@(?P.*)', string) + if m: + return m.group('file') + else: + return None + + +class Copy(LoraxAction): + + REGEX = r'^(?P.*?)\sto\s(?P.*?)(\smode\s(?P.*?))?$' + + def __init__(self, **kwargs): + LoraxAction.__init__(self) + self._attrs['src'] = kwargs.get('src') + self._attrs['dst'] = kwargs.get('dst') + self._attrs['mode'] = kwargs.get('mode') + + file = getFileName(self._attrs['src']) + if file: + self._attrs['install'] = file + + def execute(self, verbose=False): + cp(src=self.src, dst=self.dst, mode=self.mode, verbose=verbose) + self._attrs['success'] = True + + def getDeps(self): + return self._attrs['src'] + + @property + def src(self): + return self._attrs['src'] + + @property + def dst(self): + return self._attrs['dst'] + + @property + def mode(self): + return self._attrs['mode'] + + @property + def install(self): + return self._attrs.get('install') + + +class Move(Copy): + def execute(self, verbose=False): + mv(src=self.src, dst=self.dst, mode=self.mode, verbose=verbose) + self._attrs['success'] = True + + +class Link(LoraxAction): + + REGEX = r'^(?P.*?)\sto\s(?P.*?)$' + + def __init__(self, **kwargs): + LoraxAction.__init__(self) + self._attrs['name'] = kwargs.get('name') + self._attrs['target'] = kwargs.get('target') + + file = getFileName(self._attrs['name']) + if file: + self._attrs['install'] = file + + def execute(self, verbose=False): + os.symlink(self.name, self.target) + self._attrs['success'] = True + + @property + def name(self): + return self._attrs['name'] + + @property + def target(self): + return self._attrs['target'] + + @property + def install(self): + return self._attrs['install'] + + +class Touch(LoraxAction): + + REGEX = r'^(?P.*?)$' + + def __init__(self, **kwargs): + LoraxAction.__init__(self) + self._attrs['filename'] = kwargs.get('filename') + + def execute(self, verbose=False): + touch(filename=self.filename, verbose=verbose) + self._attrs['success'] = True + + @property + def filename(self): + return self._attrs['filename'] + + +class Edit(Touch): + + REGEX = r'^(?P.*?)\stext\s"(?P.*?)"((?P\sappend?))?$' + + def __init__(self, **kwargs): + Touch.__init__(self, **kwargs) + self._attrs['text'] = kwargs.get('text') + + append = kwargs.get('append', False) + if append: + self._attrs['append'] = True + else: + self._attrs['append'] = False + + file = getFileName(self._attrs['filename']) + if file: + self._attrs['install'] = file + + def execute(self, verbose=False): + edit(filename=self.filename, text=self.text, append=self.append, verbose=verbose) + self._attrs['success'] = True + + @property + def text(self): + return self._attrs['text'] + + @property + def append(self): + return self._attrs['append'] + + @property + def install(self): + return self._attrs['install'] + + +class Replace(Touch): + + REGEX = r'^(?P.*?)\sfind\s"(?P.*?)"\sreplace\s"(?P.*?)"$' + + def __init__(self, **kwargs): + Touch.__init__(self, **kwargs) + self._attrs['find'] = kwargs.get('find') + self._attrs['replace'] = kwargs.get('replace') + + file = getFileName(self._attrs['filename']) + if file: + self._attrs['install'] = file + + def execute(self, verbose=False): + replace(filename=self.filename, find=self.find, replace=self.replace, verbose=verbose) + self._attrs['success'] = True + + @property + def find(self): + return self._attrs['find'] + + @property + def replace(self): + return self._attrs['replace'] + + @property + def install(self): + return self._attrs['install'] diff --git a/src/pylorax/base.py b/src/pylorax/base.py new file mode 100644 index 00000000..8eca6bca --- /dev/null +++ b/src/pylorax/base.py @@ -0,0 +1,43 @@ +# pylorax/base.py + +import commands + + +class LoraxAction(object): + + REGEX = r'.*' + + def __init__(self): + if self.__class__ is LoraxAction: + raise TypeError, 'LoraxAction is an abstract class' + + self._attrs = {} + self._attrs['success'] = None + + def __str__(self): + return '%s: %s' % (self.__class__.__name__, self._attrs) + + def execute(self, verbose=False): + raise NotImplementedError, 'execute method not implemented for LoraxAction class' + + @property + def success(self): + return self._attrs['success'] + + +def seq(arg): + if type(arg) not in (type([]), type(())): + return [arg] + else: + return arg + + +def getConsoleSize(): + err, output = commands.getstatusoutput('stty size') + if not err: + height, width = output.split() + else: + # set defaults + height, width = 24, 80 + + return int(height), int(width) diff --git a/src/pylorax/config.py b/src/pylorax/config.py new file mode 100644 index 00000000..7390060e --- /dev/null +++ b/src/pylorax/config.py @@ -0,0 +1,125 @@ +# pylorax/config.py + +import sys + +from base import seq +import re + +from exceptions import TemplateError + + +class Container(object): + def __init__(self, attrs=None): + self.__dict__['__internal'] = {} + self.__dict__['__internal']['attrs'] = set() + + if attrs: + self.addAttr(attrs) + + def __str__(self): + return str(self.__makeDict()) + + def __iter__(self): + return iter(self.__makeDict()) + + def __getitem__(self, attr): + self.__checkInternal(attr) + if attr not in self.__dict__: + raise AttributeError, "'Container' object has no attribute '%s'" % attr + + return self.__dict__[attr] + + def __setattr__(self, attr, value): + raise AttributeError, 'you cannot do that, use addAttr() and set() instead' + + def __delattr__(self, attr): + raise AttributeError, 'you cannot do that, use delAttr() instead' + + def addAttr(self, attrs): + for attr in filter(lambda attr: attr not in self.__dict__, seq(attrs)): + self.__checkInternal(attr) + + self.__dict__[attr] = None + self.__dict__['__internal']['attrs'].add(attr) + + def delAttr(self, attrs): + for attr in filter(lambda attr: attr in self.__dict__, seq(attrs)): + self.__checkInternal(attr) + + del self.__dict__[attr] + self.__dict__['__internal']['attrs'].discard(attr) + + def set(self, **kwargs): + unknown = set() + for attr, value in kwargs.items(): + self.__checkInternal(attr) + + if attr in self.__dict__: + self.__dict__[attr] = value + else: + unknown.add(attr) + + return unknown + + def __makeDict(self): + d = {} + for attr in self.__dict__['__internal']['attrs']: + d[attr] = self.__dict__[attr] + + return d + + def __checkInternal(self, attr): + if attr.startswith('__'): + raise AttributeError, 'do not mess with internal stuff' + + +class Template(object): + def __init__(self): + self._actions = [] + + def parse(self, filename, supported_actions): + try: + f = open(filename, 'r') + except IOError: + sys.stdout.write('ERROR: Unable to open the template file\n') + return False + else: + lines = f.readlines() + f.close() + + active_action = '' + in_action = False + for lineno, line in enumerate(lines, start=1): + line = line.strip() + + if not line or line.startswith('#'): + continue + + if in_action and not line.startswith(':'): + # create the action object + regex = supported_actions[active_action].REGEX + m = re.match(regex, line) + if m: + new_action = supported_actions[active_action](**m.groupdict()) + self._actions.append(new_action) + else: + # didn't match the regex + raise TemplateError, 'invalid action format "%s" on line %d' % (line, lineno) + + if in_action and line.startswith(':'): + in_action = False + + if not in_action and line.startswith(':'): + active_action = line[1:] + + if active_action not in supported_actions: + raise TemplateError, 'unknown action "%s" on line %d' % (active_action, lineno) + else: + in_action = True + continue + + return True + + @property + def actions(self): + return self._actions diff --git a/src/pylorax/exceptions.py b/src/pylorax/exceptions.py new file mode 100644 index 00000000..ae402ed2 --- /dev/null +++ b/src/pylorax/exceptions.py @@ -0,0 +1,7 @@ +# pylorax/exceptions.py + +class LoraxError(Exception): + pass + +class TemplateError(Exception): + pass diff --git a/src/pylorax/fileutils.py b/src/pylorax/fileutils.py deleted file mode 100644 index ec1e1edb..00000000 --- a/src/pylorax/fileutils.py +++ /dev/null @@ -1,50 +0,0 @@ -import os -import shutil -import glob - - -def cp(src, dst, verbose=False): - for name in glob.iglob(src): - __copy(name, dst, verbose=verbose) - -def mv(src, dst, verbose=False): - for name in glob.iglob(src): - __copy(name, dst, verbose=verbose, remove=True) - - -def __copy(src, dst, verbose=False, remove=False): - if not os.path.exists(src): - print('cannot stat "%s": No such file or directory' % (src,)) - return - - if os.path.isdir(dst): - basename = os.path.basename(src) - dst = os.path.join(dst, basename) - - if os.path.isdir(src): - if os.path.isfile(dst): - print('omitting directory "%s"' % (src,)) - return - - if not os.path.isdir(dst): - os.makedirs(dst) - - names = map(lambda name: os.path.join(src, name), os.listdir(src)) - for name in names: - __copy(name, dst, verbose=verbose, remove=remove) - else: - if os.path.isdir(dst): - print('cannot overwrite directory "%s" with non-directory' % (dst,)) - return - - try: - if verbose: - print('copying "%s" to "%s"' % (src, dst)) - shutil.copy2(src, dst) - except shutil.Error, IOError: - print('cannot copy "%s" to "%s"' % (src, dst)) - else: - if remove: - if verbose: - print('removing "%s"' % (src,)) - os.unlink(src) diff --git a/src/pylorax/images.py b/src/pylorax/images.py index 3394cb80..a7af93f1 100644 --- a/src/pylorax/images.py +++ b/src/pylorax/images.py @@ -1,102 +1,140 @@ -# -# pylorax images module -# Install image and tree support data generation tool -- Python module -# +# pylorax/images.py -import datetime +import sys +import os + +from utils.fileutil import cp, mv, rm, touch, replace + +import initrd -class Images(): +class Images(object): + def __init__(self, config, yum): + self.conf = config + self.yum = yum - def __init__(self, conf, yumconf, arch, imgdir, product, version, bugurl, output, noiso=False): - self.conf = conf - self.yumconf = yumconf + # XXX don't see this used anywhere... maybe in some other script, have to check... + #syslinux = os.path.join(self.conf.treedir, 'usr', 'lib', 'syslinux', 'syslinux-nomtools') + #if not os.path.isfile(syslinux): + # print('WARNING: %s does not exist' % syslinux) + # syslinux = os.path.join(self.conf.treedir, 'usr', 'bin', 'syslinux') + # if not os.path.isfile(syslinux): + # print('ERROR: %s does not exist' % syslinux) + # sys.exit(1) - self.arch = arch - self.imgdir = imgdir - self.product = product - self.version = version - self.bugurl = bugurl + def run(self): + self.prepareBootTree() - self.output = output - self.noiso = noiso + def __makeinitrd(self, dst, size=8192, loader='loader'): + i = initrd.InitRD(self.conf, self.yum) + i.prepare() + i.processActions() + i.create(dst) + i.cleanUp() - now = datetime.datetime.now() - self.imageuuid = now.strftime('%Y%m%d%H%M') + '.' + os.uname()[4] + def prepareBootTree(self): + # install needed packages + self.yum.addPackages(['anaconda', 'anaconda-runtime', 'kernel', 'syslinux', 'memtest']) + self.yum.install() - self.initrdmods = self.__getModulesList() + # create the destination directories + self.imgdir = os.path.join(self.conf.outdir, 'images') + if os.path.exists(self.imgdir): + rm(self.imgdir) + self.pxedir = os.path.join(self.imgdir, 'pxeboot') + os.makedirs(self.imgdir) + os.makedirs(self.pxedir) - if self.arch == 'sparc64': - self.basearch = 'sparc' - else: - self.basearch = self.arch + # write the images/README + f = open(os.path.join(self.imgdir, 'README'), 'w') + f.write('This directory contains image files that can be used to create media\n' + 'capable of starting the %s installation process.\n\n' % self.conf.product) + f.write('The boot.iso file is an ISO 9660 image of a bootable CD-ROM. It is useful\n' + 'in cases where the CD-ROM installation method is not desired, but the\n' + 'CD-ROM\'s boot speed would be an advantage.\n\n') + f.write('To use this image file, burn the file onto CD-R (or CD-RW) media as you\n' + 'normally would.\n') + f.close() - self.libdir = 'lib' - if self.arch == 'x86_64' or self.arch =='s390x': - self.libdir = 'lib64' + # write the images/pxeboot/README + f = open(os.path.join(self.pxedir, 'README'), 'w') + f.write('The files in this directory are useful for booting a machine via PXE.\n\n') + f.write('The following files are available:\n') + f.write('vmlinuz - the kernel used for the installer\n') + f.write('initrd.img - an initrd with support for all install methods and\n') + f.write(' drivers supported for installation of %s\n' % self.conf.product) + f.close() - # explicit block size setting for some arches - # FIXME we compose ppc64-ish trees as ppc, so we have to set the "wrong" block size - # XXX i don't get this :) - self.crambs = [] - if self.arch == 'sparc64': - self.crambs = ['--blocksize', '8192'] - elif self.arch == 'sparc': - self.crambs = ['--blocksize', '4096'] + # set up some dir variables for further use + anacondadir = os.path.join(self.conf.treedir, 'usr', 'lib', 'anaconda-runtime') + bootdiskdir = os.path.join(anacondadir, 'boot') + syslinuxdir = os.path.join(self.conf.treedir, 'usr', 'lib', 'syslinux') - self.__setUpDirectories() + isolinuxbin = os.path.join(syslinuxdir, 'isolinux.bin') + if os.path.isfile(isolinuxbin): + print('Creating the isolinux directory...') + self.isodir = os.path.join(self.conf.outdir, 'isolinux') + if os.path.exists(self.isodir): + rm(self.isodir) + os.makedirs(self.isodir) + + # copy the isolinux.bin to isolinux dir + cp(isolinuxbin, self.isodir) + + # copy the syslinux.cfg to isolinux/isolinux.cfg + isolinuxcfg = os.path.join(self.isodir, 'isolinux.cfg') + cp(os.path.join(bootdiskdir, 'syslinux.cfg'), isolinuxcfg) - def __getModulesList(self): - modules = set() + # set the product and version in isolinux.cfg + replace(isolinuxcfg, r'@PRODUCT@', self.conf.product) + replace(isolinuxcfg, r'@VERSION@', self.conf.version) + + # copy the grub.conf to isolinux dir + cp(os.path.join(bootdiskdir, 'grub.conf'), self.isodir) - modules_files = [] - modules_files.append(os.path.join(self.conf['confdir'], 'modules')) - modules_files.append(os.path.join(self.conf['confdir'], self.arch, 'modules')) + # create the initrd in isolinux dir + initrd = os.path.join(self.isodir, 'initrd.img') + self.__makeinitrd(initrd) + + # copy the vmlinuz to isolinux dir + vmlinuz = os.path.join(self.conf.treedir, 'boot', 'vmlinuz-*') + cp(vmlinuz, os.path.join(self.isodir, 'vmlinuz')) - for pfile in modules_files: - if os.path.isfile(pfile): - f = open(pfile, 'r') - for line in f.readlines(): - line = line.strip() + # copy the splash files to isolinux dir + vesasplash = os.path.join(anacondadir, 'syslinux-vesa-splash.jpg') + if os.path.isfile(vesasplash): + cp(vesasplash, os.path.join(self.isodir, 'splash.jpg')) + vesamenu = os.path.join(syslinuxdir, 'vesamenu.c32') + cp(vesamenu, self.isodir) + replace(isolinuxcfg, r'default linux', r'default vesamenu.c32') + replace(isolinuxcfg, r'prompt 1', r'#prompt 1') + else: + splashtools = os.path.join(anacondadir, 'splashtools.sh') + splashlss = os.path.join(bootdiskdir, 'splash.lss') + if os.path.isfile(splashtools): + os.system('%s %s %s' % (splashtools, + os.path.join(bootdiskdir, 'syslinux-splash.jpg'), + splashlss)) + if os.path.isfile(splashlss): + cp(splashlss, self.isodir) - if not line or line.startswith('#'): - continue - - if line.startswith('-'): - modules.discard(line[1:]) - else: - modules.add(line) + # copy the .msg files to isolinux dir + for file in os.listdir(bootdiskdir): + if file.endswith('.msg'): + cp(os.path.join(bootdiskdir, file), self.isodir) + replace(os.path.join(self.isodir, file), r'@VERSION@', self.conf.version) + # if present, copy the memtest to isolinux dir + # XXX search for it in bootdiskdir or treedir/install/boot ? + #cp(os.path.join(bootdiskdir, 'memtest*'), os.path.join(self.isodir, 'memtest')) + cp(os.path.join(self.conf.treedir, 'boot', 'memtest*'), + os.path.join(self.isodir, 'memtest')) + if os.path.isfile(os.path.join(self.isodir, 'memtest')): + f = open(isolinuxcfg, 'a') + f.write('label memtest86\n') + f.write(' menu label ^Memory test\n') + f.write(' kernel memtest\n') + f.write(' append -\n') f.close() - - modules = list(modules) - modules.sort() - - return modules - - def __setUpDirectories(self): - imagepath = os.path.join(self.output, 'images') - fullmodpath = tempfile.mkdtemp('XXXXXX', 'instimagemods.', self.conf['tmpdir']) - finalfullmodpath = os.path.join(self.output, 'modules') - - kernelbase = tempfile.mkdtemp('XXXXXX', 'updboot.kernel.', self.conf['tmpdir']) - kernelname = 'vmlinuz' - - kerneldir = '/boot' - if self.arch == 'ia64': - kerneldir = os.path.join(kerneldir, 'efi', 'EFI', 'redhat') - - for dir in [imagepath, fullmodpath, finalfullmodpath, kernelbase]: - if os.path.isdir(dir): - shutil.rmtree(dir) - os.makedirs(dir) - - self.imagepath = imagepath - self.fullmodpath = fullmodpath - self.finalfullmodpath = finalfullmodpath - - self.kernelbase = kernelbase - self.kernelname = kernelname - self.kerneldir = kerneldir - - + else: + print('No isolinux binary found, skipping isolinux creation') diff --git a/src/pylorax/initrd.py b/src/pylorax/initrd.py new file mode 100644 index 00000000..c67ce682 --- /dev/null +++ b/src/pylorax/initrd.py @@ -0,0 +1,74 @@ +# pylorax/initrd.py + +import os +import re + +import actions +from config import Template +from utils.libutil import LDD +from utils.fileutil import rm + + +class InitRD(object): + def __init__(self, config, yum): + self.conf = config + self.yum = yum + + self.initrddir = os.path.join(self.conf.tempdir, 'initrd') + os.makedirs(self.initrddir) + + # get supported actions + supported_actions = actions.getActions() + + initrd_templates = [] + initrd_templates.append(os.path.join(self.conf.confdir, 'templates', 'initrd')) + initrd_templates.append(os.path.join(self.conf.confdir, 'templates', self.conf.buildarch, + 'initrd')) + + self.template = Template() + for file in initrd_templates: + if os.path.isfile(file): + self.template.parse(file, supported_actions) + + self.actions = [] + + def prepare(self): + # install needed packages + for action in filter(lambda action: hasattr(action, 'install'), self.template.actions): + self.yum.addPackages(action.install) + + self.yum.install() + + # get needed dependencies + ldd = LDD(libroot=os.path.join(self.conf.treedir, self.conf.libdir)) + for action in filter(lambda action: hasattr(action, 'getDeps'), self.template.actions): + file = re.sub(r'@instroot@(?P.*)', '%s\g' % self.conf.treedir, + action.getDeps()) + ldd.getDeps(file) + + # resolve symlinks + ldd.getLinks() + + # add dependencies to actions + for dep in ldd.deps: + kwargs = {} + kwargs['src'] = dep + kwargs['dst'] = re.sub(r'%s(?P.*)' % self.conf.treedir, + '%s\g' % self.initrddir, + dep) + + new_action = actions.fileactions.Copy(**kwargs) + self.actions.append(new_action) + + def processActions(self): + for action in self.template.actions: + action.execute() + + for action in self.actions: + action.execute() + + def create(self, dst): + os.system('find %s | cpio --quiet -c -o | gzip -9 > %s' % (self.initrddir, dst)) + + def cleanUp(self): + rm(self.initrddir) diff --git a/src/pylorax/utils/__init__.py b/src/pylorax/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/pylorax/utils/fileutil.py b/src/pylorax/utils/fileutil.py new file mode 100644 index 00000000..d8629de0 --- /dev/null +++ b/src/pylorax/utils/fileutil.py @@ -0,0 +1,104 @@ +# pylorax/utils/fileutil.py + +import sys +import os +import shutil +import glob +import fileinput +import re + + +def cp(src, dst, mode=None, verbose=False): + for name in glob.iglob(src): + __copy(name, dst, verbose=verbose) + if mode: + os.chmod(dst, mode) + +def mv(src, dst, mode=None, verbose=False): + for name in glob.iglob(src): + __copy(name, dst, verbose=verbose, remove=True) + if mode: + os.chmod(dst, mode) + +def rm(target, verbose=False): + if os.path.isdir(target): + if verbose: + print('removing directory "%s"' % target) + shutil.rmtree(target, ignore_errors=True) + else: + if verbose: + print('removing file "%s"' % target) + os.unlink(target) + +def __copy(src, dst, verbose=False, remove=False): + if not os.path.exists(src): + sys.stderr.write('cannot stat "%s": No such file or directory\n' % src) + return + + if os.path.isdir(dst): + basename = os.path.basename(src) + dst = os.path.join(dst, basename) + + if os.path.isdir(src): + if os.path.isfile(dst): + sys.stderr.write('omitting directory "%s"\n' % src) + return + + if not os.path.isdir(dst): + os.makedirs(dst) + + names = map(lambda name: os.path.join(src, name), os.listdir(src)) + for name in names: + __copy(name, dst, verbose=verbose, remove=remove) + else: + if os.path.isdir(dst): + sys.stderr.write('cannot overwrite directory "%s" with non-directory\n' % dst) + return + + try: + if verbose: + print('copying "%s" to "%s"' % (src, dst)) + shutil.copy2(src, dst) + except (shutil.Error, IOError) as why: + sys.stderr.write('cannot copy "%s" to "%s": %s\n' % (src, dst, why)) + else: + if remove: + if verbose: + print('removing "%s"' % src) + os.unlink(src) + + +def touch(filename, verbose=False): + if os.path.exists(filename): + os.utime(filename, None) + return True + + try: + f = open(filename, 'w') + except IOError: + return False + else: + f.close() + return True + +def edit(filename, text, append=False, verbose=False): + mode = 'w' + if append: + mode = 'a' + + try: + f = open(filename, mode) + except IOError: + return False + else: + f.write(text) + f.close() + return True + +def replace(filename, find, replace, verbose=False): + fin = fileinput.input(filename, inplace=1) + for line in fin: + line = re.sub(find, replace, line) + sys.stdout.write(line) + fin.close() + return True diff --git a/src/pylorax/utils/libutil.py b/src/pylorax/utils/libutil.py new file mode 100644 index 00000000..ea746e5b --- /dev/null +++ b/src/pylorax/utils/libutil.py @@ -0,0 +1,57 @@ +import sys + +import os +import commands +import re + + +class LDD(object): + def __init__(self, libroot='/'): + f = open('/usr/bin/ldd', 'r') + for line in f.readlines(): + line = line.strip() + if line.startswith('RTLDLIST='): + rtldlist, sep, ld_linux = line.partition('=') + break + f.close() + + self._ldd = '%s --list --library-path %s' % (ld_linux, libroot) + self._deps = set() + + def getDeps(self, filename): + rc, output = commands.getstatusoutput('%s %s' % (self._ldd, filename)) + + if rc: + return + + lines = output.splitlines() + for line in lines: + line = line.strip() + + m = re.match(r'^[a-zA-Z0-9.]*\s=>\s(?P[a-zA-Z0-9./]*)\s\(0x[0-9a-f]*\)$', line) + if m: + lib = m.group('lib') + if lib not in self._deps: + self._deps.add(lib) + self.getDeps(lib) + + def getLinks(self): + targets = set() + for lib in self._deps: + if os.path.islink(lib): + targets.add(os.path.realpath(lib)) + + self._deps.update(targets) + + @property + def deps(self): + return self._deps + + +if __name__ == '__main__': + ldd = LDD(libroot=sys.argv[2]) + ldd.getDeps(sys.argv[1]) + ldd.getLinks() + + for dep in ldd.deps: + print dep diff --git a/src/pylorax/utils/rpmutil.py b/src/pylorax/utils/rpmutil.py new file mode 100644 index 00000000..4dcf941e --- /dev/null +++ b/src/pylorax/utils/rpmutil.py @@ -0,0 +1,158 @@ +# pylorax/utils/rpmutil.py + +import sys +import os +import stat +import yum +import urlgrabber +import shutil + +import yum.callbacks +import yum.rpmtrans + +from rpmUtils.miscutils import rpm2cpio +from cpioarchive import CpioArchive + +from pylorax.base import seq, getConsoleSize + + +class Callback(yum.rpmtrans.SimpleCliCallBack): + def __init__(self): + yum.rpmtrans.SimpleCliCallBack.__init__(self) + self.height, self.width = getConsoleSize() + + def event(self, package, action, te_current, te_total, ts_current, ts_total): + # XXX crazy output stuff + progress = float(te_current) / float(te_total) + + percentage = int(progress * 100) + + bar_length = 20 + bar = int(percentage / (100/bar_length)) + + total_progress_str = '[%s/%s] ' % (ts_current, ts_total) + package_progress_str = ' [%s%s] %3d%%' % ('#' * bar, '-' * (bar_length - bar), percentage) + + action_str = '%s %s' % (self.action[action], package) + chars_left = self.width - len(total_progress_str) - len(package_progress_str) + + if len(action_str) > chars_left: + action_str = action_str[:chars_left-3] + action_str = action_str + '...' + else: + action_str = action_str + ' ' * (chars_left - len(action_str)) + + msg = total_progress_str + action_str + package_progress_str + + sys.stdout.write(msg) + sys.stdout.write('\b' * len(msg)) + + if percentage == 100: + sys.stdout.write('\n') + + sys.stdout.flush() + + +class Yum(object): + def __init__(self, yumconf='/etc/yum/yum.conf', installroot='/'): + self.yb = yum.YumBase() + + self.yumconf = os.path.abspath(yumconf) + self.installroot = os.path.abspath(installroot) + + self.yb.preconf.fn = self.yumconf + self.yb.preconf.root = self.installroot + self.yb._getConfig() + + self.yb._getRpmDB() + self.yb._getRepos() + self.yb._getSacks() + + def find(self, patterns): + pl = self.yb.doPackageLists(patterns=seq(patterns)) + return pl.installed, pl.available + + def isInstalled(self, pattern): + print('searching for package matching %s' % pattern) + pl = self.yb.doPackageLists(pkgnarrow='installed', patterns=[pattern]) + print('found %s' % pl.installed) + return pl.installed + + def download(self, packages): + for package in seq(packages): + print('Downloading package %s...' % package) + fn = urlgrabber.urlgrab(package.remote_url) + shutil.copy(fn, self.installroot) + + return os.path.join(self.installroot, os.path.basename(fn)) + + def addPackages(self, patterns): + # FIXME don't add packages already installed + for pattern in seq(patterns): + installed = self.isInstalled(pattern) + if installed: + print 'Package %s already installed' % installed + return + + print('Adding package matching %s...' % pattern) + try: + self.yb.install(name=pattern) + except yum.Errors.InstallError: + try: + self.yb.install(pattern=pattern) + except yum.Errors.InstallError: + sys.stderr.write('ERROR: No package matching %s available\n' % pattern) + + def install(self): + self.yb.resolveDeps() + self.yb.buildTransaction() + + cb = yum.callbacks.ProcessTransBaseCallback() + rpmcb = Callback() + self.yb.processTransaction(callback=cb, rpmDisplay=rpmcb) + + +def extract_rpm(rpmfile, destdir): + if not os.path.isdir(destdir): + os.makedirs(destdir) + + rpm = os.open(rpmfile, os.O_RDONLY) + output = open(os.path.join(destdir, 'CONTENT.cpio'), 'w') + + rpm2cpio(rpm, output) + output.close() + + cwd = os.getcwd() + os.chdir(destdir) + + cpio = CpioArchive(name=output.name) + for entry in cpio: + path = os.path.abspath(entry.name) + isdir = stat.S_ISDIR(entry.mode) + + if isdir: + if not os.path.isdir(path): + os.makedirs(path) + else: + print('Extracting %s...' % entry.name) + dir = os.path.dirname(path) + if not os.path.isdir(dir): + os.makedirs(dir) + + try: + f = open(path, 'w') + except IOError: + sys.stderr.write('ERROR: Unable to extract file %s\n' % path) + else: + f.write(entry.read()) + f.close() + + os.chmod(path, entry.mode) + os.chown(path, entry.uid, entry.gid) + + cpio.close() + os.unlink(output.name) + + os.chdir(cwd) + + return True