pungi/pungi/gather.py

1901 lines
76 KiB
Python
Raw Normal View History

2015-02-10 13:19:34 +00:00
# -*- coding: utf-8 -*-
2012-11-12 14:59:02 +00:00
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <https://gnu.org/licenses/>.
2012-11-12 14:59:02 +00:00
import logging
import os
import re
import shutil
import subprocess
import sys
from fnmatch import fnmatch
import lockfile
import urlgrabber.progress
import yum
from productmd.common import SortedConfigParser
import ConfigParser
2012-11-12 14:59:02 +00:00
import arch as arch_module
import multilib_yum as multilib
import pungi.util
from pungi.wrappers.createrepo import CreaterepoWrapper
class ReentrantYumLock(object):
""" A lock that can be acquired multiple times by the same process. """
def __init__(self, lock, log):
self.lock = lock
self.log = log
self.count = 0
def __enter__(self):
if not self.count:
self.log.info("Waiting on %r" % self.lock.lock_file)
self.lock.acquire()
self.log.info("Got %r" % self.lock.lock_file)
self.count = self.count + 1
self.log.info("Lock count upped to %i" % self.count)
def __exit__(self, type, value, tb):
self.count = self.count - 1
self.log.info("Lock count downed to %i" % self.count)
self.log.info("%r %r %r" % (type, value, tb))
if not self.count:
self.lock.release()
self.log.info("Released %r" % self.lock.lock_file)
def yumlocked(method):
""" A locking decorator. """
def wrapper(self, *args, **kwargs):
with self.yumlock:
return method(self, *args, **kwargs)
# TODO - replace argspec, signature, etc..
return wrapper
2012-09-25 20:15:35 +00:00
def is_source(po):
if po.arch in ("src", "nosrc"):
return True
return False
def is_noarch(po):
if po.arch == "noarch":
return True
return False
2012-09-25 20:15:35 +00:00
def is_package(po):
if pungi.util.pkg_is_debug(po):
2012-09-25 20:15:35 +00:00
return False
if is_source(po):
return False
return True
FLAGS = {
'EQ': '=',
'GE': '>=',
'LE': '<=',
'GT': '>',
'LT': '<',
}
class Req(object):
"""A wrapper for a tuple representing a Requires tag.
Only useful for formatting the value into a human readable string.
"""
def __init__(self, req):
self.r, self.f, self.v = req
def __str__(self):
if self.f and self.v:
flag = FLAGS.get(self.f, '??')
version = '%s:%s-%s' % self.v
return '%s %s %s' % (self.r, flag, version)
return self.r
class PungiBase(object):
"""The base Pungi class. Set up config items and logging here"""
def __init__(self, config):
self.config = config
multilib.init(self.config.get('pungi', 'multilibconf'))
2012-11-12 14:59:02 +00:00
# ARCH setup
self.tree_arch = self.config.get('pungi', 'arch')
self.yum_arch = arch_module.tree_arch_to_yum_arch(self.tree_arch)
2012-11-12 14:59:02 +00:00
full_archlist = self.config.getboolean('pungi', 'full_archlist')
self.valid_arches = arch_module.get_valid_arches(self.tree_arch, multilib=full_archlist)
self.valid_arches.append("src") # throw source in there, filter it later
self.valid_native_arches = arch_module.get_valid_arches(self.tree_arch, multilib=False)
2012-11-12 14:59:02 +00:00
self.valid_multilib_arches = arch_module.get_valid_multilib_arches(self.tree_arch)
2012-09-25 20:15:35 +00:00
# arch: compatible arches
self.compatible_arches = {}
for i in self.valid_arches:
self.compatible_arches[i] = arch_module.get_compatible_arches(i)
2012-11-12 14:59:02 +00:00
self.doLoggerSetup()
self.workdir = os.path.join(self.config.get('pungi', 'workdirbase'),
self.config.get('pungi', 'variant'),
2012-11-12 14:59:02 +00:00
self.tree_arch)
def doLoggerSetup(self):
"""Setup our logger"""
logdir = os.path.join(self.config.get('pungi', 'destdir'), 'logs')
2015-02-05 15:56:24 +00:00
pungi.util._ensuredir(logdir, None, force=True) # Always allow logs to be written out
if self.config.get('pungi', 'variant'):
logfile = os.path.join(logdir, '%s.%s.log' % (self.config.get('pungi', 'variant'),
2012-11-12 14:59:02 +00:00
self.tree_arch))
else:
2012-11-12 14:59:02 +00:00
logfile = os.path.join(logdir, '%s.log' % (self.tree_arch))
# Create the root logger, that will log to our file
logging.basicConfig(level=logging.DEBUG,
format='%(name)s.%(levelname)s: %(message)s',
filename=logfile)
class CallBack(urlgrabber.progress.TextMeter):
"""A call back function used with yum."""
def __init__(self, logger):
self.logger = logger
def start(self, filename=None, url=None, basename=None, size=None, now=None, text=None):
self.logger.info('Downloading %s (%sB)'
% (text, urlgrabber.progress.format_number(size)))
def update(self, amount_read, name=None):
return
def end(self, amount_read, now=None):
return
class PungiYum(yum.YumBase):
"""Subclass of Yum"""
def __init__(self, config):
self.pungiconfig = config
yum.YumBase.__init__(self)
def doLoggingSetup(self, debuglevel, errorlevel, syslog_ident=None, syslog_facility=None):
"""Setup the logging facility."""
logdir = os.path.join(self.pungiconfig.get('pungi', 'destdir'), 'logs')
if not os.path.exists(logdir):
os.makedirs(logdir)
if self.pungiconfig.get('pungi', 'variant'):
logfile = os.path.join(logdir, '%s.%s.log' % (self.pungiconfig.get('pungi', 'variant'),
self.pungiconfig.get('pungi', 'arch')))
else:
logfile = os.path.join(logdir, '%s.log' % (self.pungiconfig.get('pungi', 'arch')))
yum.logging.basicConfig(level=yum.logging.DEBUG, filename=logfile)
def doFileLogSetup(self, uid, logfile):
# This function overrides a yum function, allowing pungi to control
# the logging.
pass
def _compare_providers(self, *args, **kwargs):
# HACK: always prefer 64bit over 32bit packages
result = yum.YumBase._compare_providers(self, *args, **kwargs)
if len(result) >= 2:
pkg1 = result[0][0]
pkg2 = result[1][0]
if pkg1.name == pkg2.name:
best_arch = self.arch.get_best_arch_from_list([pkg1.arch, pkg2.arch], self.arch.canonarch)
if best_arch != "noarch" and best_arch != pkg1.arch:
result[0:1] = result[0:1:-1]
return result
2015-02-05 15:56:24 +00:00
class Pungi(PungiBase):
def __init__(self, config, ksparser):
2015-02-05 15:56:24 +00:00
PungiBase.__init__(self, config)
# Set our own logging name space
self.logger = logging.getLogger('Pungi')
# Create a lock object for later use.
filename = self.config.get('pungi', 'cachedir') + "/yumlock"
lock = lockfile.LockFile(filename)
self.yumlock = ReentrantYumLock(lock, self.logger)
if not self.logger.handlers:
# Create the stdout/err streams and only send INFO+ stuff there
formatter = logging.Formatter('%(name)s:%(levelname)s: %(message)s')
console = logging.StreamHandler()
console.setFormatter(formatter)
console.setLevel(logging.INFO)
self.logger.addHandler(console)
self.destdir = self.config.get('pungi', 'destdir')
self.archdir = os.path.join(self.destdir,
self.config.get('pungi', 'version'),
self.config.get('pungi', 'variant'),
2012-11-12 14:59:02 +00:00
self.tree_arch)
self.topdir = os.path.join(self.archdir, 'os')
self.isodir = os.path.join(self.archdir, self.config.get('pungi','isodir'))
2015-02-05 15:56:24 +00:00
pungi.util._ensuredir(self.workdir, self.logger, force=True)
self.common_files = []
self.infofile = os.path.join(self.config.get('pungi', 'destdir'),
self.config.get('pungi', 'version'),
'.composeinfo')
self.ksparser = ksparser
self.resolved_deps = {} # list the deps we've already resolved, short circuit
self.excluded_packages = set() # set of packages we've already excluded
self.multilib_blacklist = set() # set of packages we've already excluded through a multilib blacklist
2012-10-15 15:16:54 +00:00
self.seen_pkgs = {} # list the packages we've already seen so we can check all deps only once
self.multilib_methods = self.config.get('pungi', 'multilib').split(" ")
# greedy methods:
# * none: only best match package
# * all: all packages matching a provide
# * build: best match package + all other packages from the same SRPM having the same provide
self.greedy_method = self.config.get('pungi', 'greedy')
2012-10-25 12:47:19 +00:00
self.lookaside_repos = self.config.get('pungi', 'lookaside_repos').split(" ")
2012-09-25 20:15:35 +00:00
self.sourcerpm_arch_map = {} # {sourcerpm: set[arches]} - used for gathering debuginfo
# package object lists
self.po_list = set()
self.srpm_po_list = set()
self.debuginfo_po_list = set()
# get_srpm_po() cache
self.sourcerpm_srpmpo_map = {}
# flags
self.input_packages = set() # packages specified in %packages kickstart section including those defined via comps groups
self.comps_packages = set() # packages specified in %packages kickstart section *indirectly* via comps groups
self.prepopulate_packages = set() # packages specified in %prepopulate kickstart section
self.fulltree_packages = set()
self.langpack_packages = set()
self.multilib_packages = set()
# already processed packages
self.completed_add_srpms = set() # srpms
self.completed_debuginfo = set() # rpms
self.completed_depsolve = set() # rpms
self.completed_langpacks = set() # rpms
self.completed_multilib = set() # rpms
self.completed_fulltree = set() # srpms
self.completed_selfhosting = set() # srpms
self.completed_greedy_build = set() # po.sourcerpm
self.is_fulltree = self.config.getboolean("pungi", "fulltree")
self.is_selfhosting = self.config.getboolean("pungi", "selfhosting")
self.is_sources = not self.config.getboolean("pungi", "nosource")
self.is_debuginfo = not self.config.getboolean("pungi", "nodebuginfo")
self.is_resolve_deps = self.config.getboolean("pungi", "resolve_deps")
self.is_nomacboot = self.config.getboolean("pungi", "nomacboot")
self.fulltree_excludes = set(self.ksparser.handler.fulltree_excludes)
# rootfs image size
self.rootfs_size = self.config.get('pungi', 'rootfs_size')
def _add_yum_repo(self, name, url, mirrorlist=False, groups=True,
cost=1000, includepkgs=None, excludepkgs=None,
proxy=None):
"""This function adds a repo to the yum object.
name: Name of the repo
url: Full url to the repo
mirrorlist: Bool for whether or not url is a mirrorlist
groups: Bool for whether or not to use groupdata from this repo
cost: an optional int representing the cost of a repo
includepkgs: An optional list of includes to use
excludepkgs: An optional list of excludes to use
proxy: An optional proxy to use
"""
includepkgs = includepkgs or []
excludepkgs = excludepkgs or []
self.logger.info('Adding repo %s' % name)
thisrepo = yum.yumRepo.YumRepository(name)
thisrepo.name = name
# add excludes and such here when pykickstart gets them
if mirrorlist:
thisrepo.mirrorlist = yum.parser.varReplace(url,
self.ayum.conf.yumvar)
self.mirrorlists.append(thisrepo.mirrorlist)
self.logger.info('Mirrorlist for repo %s is %s' %
(thisrepo.name, thisrepo.mirrorlist))
else:
thisrepo.baseurl = yum.parser.varReplace(url,
self.ayum.conf.yumvar)
self.repos.extend(thisrepo.baseurl)
self.logger.info('URL for repo %s is %s' %
(thisrepo.name, thisrepo.baseurl))
thisrepo.basecachedir = self.ayum.conf.cachedir
thisrepo.enablegroups = groups
# This is until yum uses this failover by default
thisrepo.failovermethod = 'priority'
thisrepo.exclude = excludepkgs
thisrepo.includepkgs = includepkgs
thisrepo.cost = cost
# Yum doesn't like proxy being None
if proxy:
thisrepo.proxy = proxy
self.ayum.repos.add(thisrepo)
self.ayum.repos.enableRepo(thisrepo.id)
self.ayum._getRepos(thisrepo=thisrepo.id, doSetup=True)
# Set the repo callback.
self.ayum.repos.setProgressBar(CallBack(logger=self.logger))
self.ayum.repos.callback = CallBack(logger=self.logger)
thisrepo.metadata_expire = 0
thisrepo.mirrorlist_expire = 0
if os.path.exists(os.path.join(thisrepo.cachedir, 'repomd.xml')):
os.remove(os.path.join(thisrepo.cachedir, 'repomd.xml'))
@yumlocked
def _inityum(self):
"""Initialize the yum object. Only needed for certain actions."""
# Create a yum object to use
self.repos = []
self.mirrorlists = []
self.ayum = PungiYum(self.config)
self.ayum.doLoggingSetup(6, 6)
yumconf = yum.config.YumConf()
yumconf.debuglevel = 6
yumconf.errorlevel = 6
yumconf.cachedir = self.config.get('pungi', 'cachedir')
yumconf.persistdir = "/var/lib/yum" # keep at default, gets appended to installroot
yumconf.installroot = os.path.join(self.workdir, 'yumroot')
yumconf.uid = os.geteuid()
yumconf.cache = 0
yumconf.failovermethod = 'priority'
2013-02-28 16:38:19 +00:00
yumconf.deltarpm = 0
yumvars = yum.config._getEnvVar()
yumvars['releasever'] = self.config.get('pungi', 'version')
2012-11-12 14:59:02 +00:00
yumvars['basearch'] = yum.rpmUtils.arch.getBaseArch(myarch=self.tree_arch)
yumconf.yumvar = yumvars
self.ayum._conf = yumconf
2010-06-29 22:58:55 +00:00
# I have no idea why this fixes a traceback, but James says it does.
del self.ayum.prerepoconf
self.ayum.repos.setCacheDir(self.ayum.conf.cachedir)
self.ayum.arch.setup_arch(self.yum_arch)
# deal with our repos
try:
self.ksparser.handler.repo.methodToRepo()
except:
pass
for repo in self.ksparser.handler.repo.repoList:
if repo.mirrorlist:
# The not bool() thing is because pykickstart is yes/no on
# whether to ignore groups, but yum is a yes/no on whether to
# include groups. Awkward.
self._add_yum_repo(repo.name, repo.mirrorlist,
mirrorlist=True,
groups=not bool(repo.ignoregroups),
cost=repo.cost,
includepkgs=repo.includepkgs,
excludepkgs=repo.excludepkgs,
proxy=repo.proxy)
else:
self._add_yum_repo(repo.name, repo.baseurl,
mirrorlist=False,
groups=not bool(repo.ignoregroups),
cost=repo.cost,
includepkgs=repo.includepkgs,
excludepkgs=repo.excludepkgs,
proxy=repo.proxy)
2012-11-12 14:59:02 +00:00
self.logger.info('Getting sacks for arches %s' % self.valid_arches)
self.ayum._getSacks(archlist=self.valid_arches)
def _filtersrcdebug(self, po):
"""Filter out package objects that are of 'src' arch."""
if po.arch == 'src' or pungi.util.pkg_is_debug(po):
return False
return True
2012-09-25 20:15:35 +00:00
def add_package(self, po, msg=None):
if not is_package(po):
raise ValueError("Not a binary package: %s" % po)
if msg:
self.logger.info(msg)
if po not in self.po_list:
self.po_list.add(po)
2012-09-25 20:15:35 +00:00
self.ayum.install(po)
self.sourcerpm_arch_map.setdefault(po.sourcerpm, set()).add(po.arch)
def add_debuginfo(self, po, msg=None):
if not pungi.util.pkg_is_debug(po):
2012-09-25 20:15:35 +00:00
raise ValueError("Not a debuginfog package: %s" % po)
if msg:
self.logger.info(msg)
if po not in self.debuginfo_po_list:
self.debuginfo_po_list.add(po)
2012-09-25 20:15:35 +00:00
def add_source(self, po, msg=None):
if not is_source(po):
raise ValueError("Not a source package: %s" % po)
if msg:
self.logger.info(msg)
if po not in self.srpm_po_list:
self.srpm_po_list.add(po)
2012-09-25 20:15:35 +00:00
def verifyCachePkg(self, po, path): # Stolen from yum
"""check the package checksum vs the cache
return True if pkg is good, False if not"""
(csum_type, csum) = po.returnIdSum()
try:
filesum = yum.misc.checksum(csum_type, path)
except yum.Errors.MiscError:
return False
if filesum != csum:
return False
return True
def expand_multilib_blacklist(self):
multilib_blacklist = self.ksparser.handler.multilib_blacklist
exactmatched, matched, unmatched = yum.packages.parsePackages(
self.all_pkgs, multilib_blacklist, casematch=1, pkgdict=self.pkg_refs.copy())
for i in sorted(unmatched):
self.logger.warning("Unmatched multilib blacklist pattern: %s" % i)
for pkg in exactmatched + matched:
if pkg.arch == "src":
continue
if pkg.arch not in self.valid_multilib_arches:
continue
found = None
for pattern in multilib_blacklist:
if fnmatch(pkg.name, pattern):
found = pattern
break
if found:
if pkg not in self.multilib_blacklist:
self.logger.info("Excluding %s.%s (multilib-blacklist pattern: %s)"
% (pkg.name, pkg.arch, found))
self.multilib_blacklist.add(pkg)
def expand_excluded_list(self):
excluded_list = []
multilib_excluded_list = []
source_excluded_list = []
for pattern in self.ksparser.handler.packages.excludedList:
if pattern.endswith(".+"):
multilib_excluded_list.append(pattern[:-2])
elif pattern.endswith(".src"):
source_excluded_list.append(pattern[:-4])
else:
excluded_list.append(pattern)
# native packages
exactmatched, matched, unmatched = yum.packages.parsePackages(
self.all_pkgs, excluded_list, casematch=1, pkgdict=self.pkg_refs.copy())
for i in sorted(unmatched):
self.logger.warning("Unmatched exclude: %s" % i)
for pkg in exactmatched + matched:
if pkg.arch == "src":
continue
if pkg.repoid in self.lookaside_repos:
# Don't exclude packages from lookaside
continue
found = None
for pattern in excluded_list:
if fnmatch(pkg.name, pattern):
found = pattern
break
if found:
if pkg not in self.excluded_packages:
self.logger.info("Excluding %s.%s (pattern: %s)"
% (pkg.name, pkg.arch, found))
self.excluded_packages.add(pkg)
# multilib packages
exactmatched, matched, unmatched = yum.packages.parsePackages(
self.all_pkgs, multilib_excluded_list, casematch=1, pkgdict=self.pkg_refs.copy())
for i in sorted(unmatched):
self.logger.warning("Unmatched multilib exclude: %s.+" % i)
for pkg in exactmatched + matched:
if pkg.arch == "src":
continue
if pkg.arch not in self.valid_multilib_arches:
continue
if pkg.repoid in self.lookaside_repos:
# Don't exclude packages from lookaside
continue
found = None
for pattern in multilib_excluded_list:
if fnmatch(pkg.name, pattern):
found = pattern
break
if found:
if pkg not in self.excluded_packages:
self.logger.info("Excluding %s.%s (pattern: %s.+)"
% (pkg.name, pkg.arch, found))
self.excluded_packages.add(pkg)
# source packages
exactmatched, matched, unmatched = yum.packages.parsePackages(
self.all_pkgs, source_excluded_list, casematch=1, pkgdict=self.pkg_refs.copy())
for i in sorted(unmatched):
self.logger.warning("Unmatched source exclude: %s.src" % i)
for pkg in exactmatched + matched:
if pkg.arch != "src":
continue
found = None
for pattern in source_excluded_list:
if fnmatch(pkg.name, pattern):
found = pattern
break
if found:
if pkg not in self.excluded_packages:
self.logger.info("Excluding %s.%s (pattern: %s.src)"
% (pkg.name, pkg.arch, found))
self.excluded_packages.add(pkg)
def excludePackages(self, pkg_sack):
"""exclude packages according to config file"""
if not pkg_sack:
return pkg_sack
result = []
for pkg in pkg_sack:
if pkg in self.multilib_blacklist:
continue
if pkg in self.excluded_packages:
continue
result.append(pkg)
return result
def get_package_deps(self, po):
"""Add the dependencies for a given package to the
transaction info"""
added = set()
if po.repoid in self.lookaside_repos:
# Don't resolve deps for stuff in lookaside.
return added
if po in self.completed_depsolve:
return added
self.completed_depsolve.add(po)
2012-10-15 15:16:54 +00:00
self.logger.info('Checking deps of %s.%s' % (po.name, po.arch))
reqs = po.requires
provs = po.provides
2012-10-16 11:52:03 +00:00
for req in reqs:
if req in self.resolved_deps:
continue
r, f, v = req
if r.startswith('rpmlib(') or r.startswith('config('):
continue
if req in provs:
continue
try:
deps = self.ayum.whatProvides(r, f, v).returnPackages()
deps = self.excludePackages(deps)
if not deps:
self.logger.warn(
"Unresolvable dependency %s in %s.%s"
% (Req(req), po.name, po.arch)
)
continue
if self.greedy_method == "all":
deps = yum.packageSack.ListPackageSack(deps).returnNewestByNameArch()
else:
found = False
for dep in deps:
if dep in self.po_list:
# HACK: there can be builds in the input list on which we want to apply the "build" greedy rules
if self.greedy_method == "build" and dep.sourcerpm not in self.completed_greedy_build:
break
found = True
break
if found:
deps = []
else:
all_deps = deps
deps = [self.ayum._bestPackageFromList(all_deps)]
if self.greedy_method == "build":
# handle "build" greedy method
if deps:
build_po = deps[0]
if is_package(build_po):
if build_po.arch != "noarch" and build_po.arch not in self.valid_multilib_arches:
all_deps = [ i for i in all_deps if i.arch not in self.valid_multilib_arches ]
for dep in all_deps:
if dep != build_po and dep.sourcerpm == build_po.sourcerpm:
deps.append(dep)
self.completed_greedy_build.add(dep.sourcerpm)
for dep in deps:
if dep not in added:
msg = 'Added %s.%s (repo: %s) for %s.%s (Requires: %s)' % (
dep.name, dep.arch, dep.repoid, po.name, po.arch, Req(req))
2012-09-25 20:15:35 +00:00
self.add_package(dep, msg)
added.add(dep)
except (yum.Errors.InstallError, yum.Errors.YumBaseError) as ex:
self.logger.warn("Unresolvable dependency %s in %s.%s (repo: %s)" % (r, po.name, po.arch, po.repoid))
continue
self.resolved_deps[req] = None
for add in sorted(added):
self.get_package_deps(add)
return added
def add_langpacks(self, po_list=None):
po_list = po_list or self.po_list
added = set()
for po in sorted(po_list):
if po in self.completed_langpacks:
continue
2012-10-16 11:52:03 +00:00
# get all langpacks matching the package name
langpacks = [ i for i in self.langpacks if i["name"] == po.name ]
if not langpacks:
continue
self.completed_langpacks.add(po)
for langpack in langpacks:
pattern = langpack["install"] % "*" # replace '%s' with '*'
exactmatched, matched, unmatched = yum.packages.parsePackages(self.all_pkgs, [pattern], casematch=1, pkgdict=self.pkg_refs.copy())
matches = filter(self._filtersrcdebug, exactmatched + matched)
matches = [ i for i in matches if not i.name.endswith("-devel") and not i.name.endswith("-static") and i.name != "man-pages-overrides" ]
matches = [ i for i in matches if fnmatch(i.name, pattern) ]
packages_by_name = {}
for i in matches:
packages_by_name.setdefault(i.name, []).append(i)
for i, pkg_sack in packages_by_name.iteritems():
pkg_sack = self.excludePackages(pkg_sack)
if not pkg_sack:
continue
match = self.ayum._bestPackageFromList(pkg_sack)
msg = 'Added langpack %s.%s (repo: %s) for package %s (pattern: %s)' % (match.name, match.arch, match.repoid, po.name, pattern)
self.add_package(match, msg)
self.completed_langpacks.add(match) # assuming langpack doesn't have langpacks
added.add(match)
return added
2012-10-16 11:52:03 +00:00
def add_multilib(self, po_list=None):
po_list = po_list or self.po_list
added = set()
if not self.multilib_methods:
return added
for po in sorted(po_list):
if po in self.completed_multilib:
continue
if po.arch in ("noarch", "src", "nosrc"):
continue
if po.arch in self.valid_multilib_arches:
continue
self.completed_multilib.add(po)
matches = self.ayum.pkgSack.searchNevra(name=po.name, ver=po.version, rel=po.release)
matches = [i for i in matches if i.arch in self.valid_multilib_arches]
if not matches:
continue
matches = self.excludePackages(matches)
match = self.ayum._bestPackageFromList(matches)
if not match:
continue
found = False
for pattern in self.ksparser.handler.multilib_whitelist:
if fnmatch(po.name, pattern):
found = True
break
if found:
msg = "Added multilib package %s.%s (repo: %s) for package %s.%s (method: %s)" % (match.name, match.arch, match.repoid, po.name, po.arch, "multilib-whitelist")
self.add_package(match, msg)
self.completed_multilib.add(match)
added.add(match)
continue
method = multilib.po_is_multilib(po, self.multilib_methods)
if not method:
continue
msg = "Added multilib package %s.%s (repo: %s) for package %s.%s (method: %s)" % (match.name, match.arch, match.repoid, po.name, po.arch, method)
self.add_package(match, msg)
self.completed_multilib.add(match)
added.add(match)
2012-10-16 11:52:03 +00:00
return added
def getPackagesFromGroup(self, group):
"""Get a list of package names from a ksparser group object
Returns a list of package names"""
packages = []
# Check if we have the group
if not self.ayum.comps.has_group(group.name):
self.logger.error("Group %s not found in comps!" % group)
return packages
# Get the group object to work with
groupobj = self.ayum.comps.return_group(group.name)
# Add the mandatory packages
packages.extend(groupobj.mandatory_packages.keys())
# Add the default packages unless we don't want them
if group.include == 1:
packages.extend(groupobj.default_packages.keys())
# Add the optional packages if we want them
if group.include == 2:
packages.extend(groupobj.default_packages.keys())
packages.extend(groupobj.optional_packages.keys())
# Deal with conditional packages
# Populate a dict with the name of the required package and value
# of the package objects it would bring in. To be used later if
# we match the conditional.
for condreq, cond in groupobj.conditional_packages.iteritems():
matches = self.ayum.pkgSack.searchNevra(name=condreq)
if matches:
if self.greedy_method != "all":
# works for both "none" and "build" greedy methods
matches = [self.ayum._bestPackageFromList(matches)]
self.ayum.tsInfo.conditionals.setdefault(cond, []).extend(matches)
return packages
def _addDefaultGroups(self, excludeGroups=None):
"""Cycle through the groups and return at list of the ones that ara
default."""
excludeGroups = excludeGroups or []
# This is mostly stolen from anaconda.
groups = map(lambda x: x.groupid,
filter(lambda x: x.default, self.ayum.comps.groups))
2010-06-11 15:29:31 +00:00
groups = [x for x in groups if x not in excludeGroups]
2010-06-11 15:29:31 +00:00
self.logger.debug('Add default groups %s' % groups)
return groups
def get_langpacks(self):
2012-10-16 11:52:03 +00:00
try:
self.langpacks = list(self.ayum.comps.langpacks)
except AttributeError:
# old yum
self.logger.warning("Could not get langpacks via yum.comps. You may need to update yum.")
self.langpacks = []
except yum.Errors.GroupsError:
# no groups or no comps at all
self.logger.warning("Could not get langpacks due to missing comps in repodata or --ignoregroups=true option.")
self.langpacks = []
def getPackageObjects(self):
"""Cycle through the list of packages and get package object matches."""
searchlist = [] # The list of package names/globs to search for
excludeGroups = [] # A list of groups for removal defined in the ks file
# precompute pkgs and pkg_refs to speed things up
self.all_pkgs = list(set(self.ayum.pkgSack.returnPackages()))
self.pkg_refs = yum.packages.buildPkgRefDict(self.all_pkgs, casematch=True)
self.expand_excluded_list()
self.expand_multilib_blacklist()
self.all_pkgs = self.excludePackages(self.all_pkgs)
lookaside_nvrs = set()
for po in self.all_pkgs:
if po.repoid in self.lookaside_repos:
lookaside_nvrs.add(po.nvra)
all_pkgs = [] # building a new list is cheaper than deleting from existing
for po in sorted(self.all_pkgs):
if po.repoid not in self.lookaside_repos and po.nvra in lookaside_nvrs:
self.logger.info("Removed %s (repo: %s), because it's also in a lookaside repo"
% (po, po.repoid))
self.excluded_packages.add(po)
else:
all_pkgs.append(po)
self.all_pkgs = all_pkgs
self.get_langpacks()
# First remove the excludes
self.ayum.excludePackages()
# Get the groups set for removal
for group in self.ksparser.handler.packages.excludedGroupList:
excludeGroups.append(str(group)[1:])
if "core" in [ i.groupid for i in self.ayum.comps.groups ]:
if "core" not in [ i.name for i in self.ksparser.handler.packages.groupList ]:
self.logger.warning("The @core group is no longer added by default; Please add @core to the kickstart if you want it in.")
if "base" in [ i.groupid for i in self.ayum.comps.groups ]:
if "base" not in [ i.name for i in self.ksparser.handler.packages.groupList ]:
if self.ksparser.handler.packages.addBase:
self.logger.warning("The --nobase kickstart option is no longer supported; Please add @base to the kickstart if you want it in.")
# Check to see if we want all the defaults
if self.ksparser.handler.packages.default:
for group in self._addDefaultGroups(excludeGroups):
self.ksparser.handler.packages.add(['@%s' % group])
# Get a list of packages from groups
comps_package_names = set()
for group in self.ksparser.handler.packages.groupList:
comps_package_names.update(self.getPackagesFromGroup(group))
searchlist.extend(sorted(comps_package_names))
# Add packages
searchlist.extend(self.ksparser.handler.packages.packageList)
input_packages = searchlist[:]
# Add prepopulate packages
prepopulate_packages = self.ksparser.handler.prepopulate
searchlist.extend(prepopulate_packages)
# Make the search list unique
searchlist = yum.misc.unique(searchlist)
for name in searchlist:
pattern = name
multilib = False
orig_name = name
if name.endswith(".+"):
name = name[:-2]
multilib = True
if self.greedy_method == "all" and name == "system-release":
# HACK: handles a special case, when system-release virtual provide is specified in the greedy mode
matches = self.ayum.whatProvides(name, None, None).returnPackages()
else:
exactmatched, matched, unmatched = yum.packages.parsePackages(self.all_pkgs, [name], casematch=1, pkgdict=self.pkg_refs.copy())
matches = exactmatched + matched
matches = filter(self._filtersrcdebug, matches)
if multilib and self.greedy_method != "all":
matches = [ po for po in matches if po.arch in self.valid_multilib_arches ]
if not matches:
self.logger.warn('Could not find a match for %s in any configured repo' % pattern)
continue
packages_by_name = {}
for po in matches:
packages_by_name.setdefault(po.name, []).append(po)
for name, packages in packages_by_name.iteritems():
packages = self.excludePackages(packages or [])
if not packages:
continue
if self.greedy_method == "all":
packages = yum.packageSack.ListPackageSack(packages).returnNewestByNameArch()
else:
# works for both "none" and "build" greedy methods
packages = [self.ayum._bestPackageFromList(packages)]
if orig_name in input_packages:
self.input_packages.update(packages)
if name in comps_package_names:
self.comps_packages.update(packages)
for po in packages:
msg = 'Found %s.%s' % (po.name, po.arch)
self.add_package(po, msg)
name_arch = "%s.%s" % (po.name, po.arch)
if name_arch in prepopulate_packages:
self.prepopulate_packages.add(po)
self.logger.info('Finished gathering package objects.')
def gather(self):
# get package objects according to the input list
self.getPackageObjects()
if self.is_sources:
self.createSourceHashes()
pass_num = 0
added = set()
while 1:
if pass_num > 0 and not added:
break
added = set()
pass_num += 1
self.logger.info("Pass #%s" % pass_num)
if self.is_resolve_deps:
# get conditional deps (defined in comps)
for txmbr in self.ayum.tsInfo:
if not txmbr.po in self.po_list:
2013-06-20 10:45:42 +00:00
if not is_package(txmbr.po):
# we don't want sources which can be pulled in, because 'src' arch is part of self.valid_arches
continue
if not txmbr.isDep:
continue
self.add_package(txmbr.po)
# resolve deps
if self.is_resolve_deps:
for po in sorted(self.po_list):
added.update(self.get_package_deps(po))
if self.is_sources:
added_srpms = self.add_srpms()
added.update(added_srpms)
if self.is_selfhosting:
for srpm_po in sorted(added_srpms):
added.update(self.get_package_deps(srpm_po))
if self.is_fulltree:
new = self.add_fulltree()
self.fulltree_packages.update(new)
self.fulltree_packages.update([ self.sourcerpm_srpmpo_map[i.sourcerpm] for i in new ])
added.update(new)
if added:
continue
# add langpacks
new = self.add_langpacks(self.po_list)
self.langpack_packages.update(new)
if self.is_sources:
self.langpack_packages.update([ self.sourcerpm_srpmpo_map[i.sourcerpm] for i in new ])
added.update(new)
if added:
continue
# add multilib packages
new = self.add_multilib(self.po_list)
self.multilib_packages.update(new)
self.multilib_packages.update([ self.sourcerpm_srpmpo_map[i.sourcerpm] for i in new ])
added.update(new)
if added:
continue
def get_srpm_po(self, po):
"""Given a package object, get a package object for the corresponding source rpm."""
# return srpm_po from cache if available
srpm_po = self.sourcerpm_srpmpo_map.get(po.sourcerpm, None)
if srpm_po is not None:
return srpm_po
# arch can be "src" or "nosrc"
2012-12-11 12:32:50 +00:00
nvr, arch, _ = po.sourcerpm.rsplit(".", 2)
name, ver, rel = nvr.rsplit('-', 2)
# ... but even "nosrc" packages are stored as "src" in repodata
srpm_po_list = self.ayum.pkgSack.searchNevra(name=name, ver=ver, rel=rel, arch="src")
srpm_po_list = self.excludePackages(srpm_po_list)
try:
srpm_po = srpm_po_list[0]
except IndexError:
self.logger.warning("Cannot find a source rpm for %s" % po.sourcerpm)
srpm_po = None
self.sourcerpm_srpmpo_map[po.sourcerpm] = srpm_po
return srpm_po
def createSourceHashes(self):
"""Create two dicts - one that maps binary POs to source POs, and
one that maps a single source PO to all binary POs it produces.
Requires yum still configured."""
self.src_by_bin = {}
self.bin_by_src = {}
self.logger.info("Generating source <-> binary package mappings")
for po in self.all_pkgs:
if is_source(po):
continue
srpmpo = self.get_srpm_po(po)
2014-04-01 19:48:59 +00:00
self.src_by_bin[po] = srpmpo
self.bin_by_src.setdefault(srpmpo, []).append(po)
2014-04-01 19:48:59 +00:00
def add_srpms(self, po_list=None):
"""Cycle through the list of package objects and
find the sourcerpm for them. Requires yum still
configured and a list of package objects"""
srpms = set()
po_list = po_list or self.po_list
for po in sorted(po_list):
try:
srpm_po = self.sourcerpm_srpmpo_map[po.sourcerpm]
except KeyError:
self.logger.error("Cannot get source RPM '%s' for %s" % (po.sourcerpm, po.nvra))
srpm_po = None
if srpm_po is None:
continue
# flags
if po in self.input_packages:
self.input_packages.add(srpm_po)
if po in self.fulltree_packages:
self.fulltree_packages.add(srpm_po)
if po in self.langpack_packages:
self.langpack_packages.add(srpm_po)
if po in self.multilib_packages:
self.multilib_packages.add(srpm_po)
if srpm_po in self.completed_add_srpms:
continue
msg = "Added source package %s.%s (repo: %s)" % (srpm_po.name, srpm_po.arch, srpm_po.repoid)
self.add_source(srpm_po, msg)
self.completed_add_srpms.add(srpm_po)
srpms.add(srpm_po)
return srpms
def add_fulltree(self, srpm_po_list=None):
"""Cycle through all package objects, and add any
that correspond to a source rpm that we are including.
Requires yum still configured and a list of package
objects."""
self.logger.info("Completing package set")
srpm_po_list = srpm_po_list or self.srpm_po_list
srpms = []
for srpm_po in srpm_po_list:
if srpm_po in self.completed_fulltree:
continue
if srpm_po.name not in self.fulltree_excludes:
srpms.append(srpm_po)
self.completed_fulltree.add(srpm_po)
added = set()
for srpm_po in srpms:
if srpm_po.repoid in self.lookaside_repos:
# Don't run fulltree on packages in lookaside
continue
include_native = False
include_multilib = False
has_native = False
has_multilib = False
for po in self.excludePackages(self.bin_by_src[srpm_po]):
if not is_package(po):
continue
if po.arch == "noarch":
continue
if po not in self.po_list:
# process only already included packages
if po.arch in self.valid_multilib_arches:
has_multilib = True
elif po.arch in self.valid_native_arches:
has_native = True
continue
if po.arch in self.valid_multilib_arches and (po in self.input_packages or self.greedy_method == "all"):
include_multilib = True
elif po.arch in self.valid_native_arches:
include_native = True
# XXX: this is very fragile!
# Do not make any changes unless you really know what you're doing!
if not include_native:
# if there's no native package already pulled in...
if has_native and not include_multilib:
# include all native packages, but only if we're not pulling multilib already
# SCENARIO: a noarch package was already pulled in and there are x86_64 and i686 packages -> we want x86_64 in to complete the package set
include_native = True
elif has_multilib:
# SCENARIO: a noarch package was already pulled in and there are no x86_64 packages; we want i686 in to complete the package set
include_multilib = True
for po in self.excludePackages(self.bin_by_src[srpm_po]):
if not is_package(po):
continue
if po in self.po_list:
continue
if po.arch != "noarch":
if po.arch in self.valid_multilib_arches:
if not include_multilib:
continue
if po.arch in self.valid_native_arches:
if not include_native:
continue
msg = "Added %s.%s (repo: %s) to complete package set" % (po.name, po.arch, po.repoid)
self.add_package(po, msg)
return added
2012-09-25 20:15:35 +00:00
def getDebuginfoList(self):
"""Cycle through the list of package objects and find
debuginfo rpms for them. Requires yum still
configured and a list of package objects"""
added = set()
for po in self.all_pkgs:
if not pungi.util.pkg_is_debug(po):
2012-09-25 20:15:35 +00:00
continue
if po.sourcerpm not in self.sourcerpm_arch_map:
# TODO: print a warning / throw an error
continue
if po.arch != 'noarch' and not (set(self.compatible_arches[po.arch]) &
set(self.sourcerpm_arch_map[po.sourcerpm]) -
set(["noarch"])):
# skip all incompatible arches unless it's a noarch debuginfo
2012-09-25 20:15:35 +00:00
# this pulls i386 debuginfo for a i686 package for example
continue
msg = 'Added debuginfo %s.%s (repo: %s)' % (po.name, po.arch, po.repoid)
2012-09-25 20:15:35 +00:00
self.add_debuginfo(po, msg)
# flags
try:
srpm_po = self.sourcerpm_srpmpo_map[po.sourcerpm]
except:
self.logger.warning('Failed to find source for %s', po.sourcerpm)
srpm_po = None
if srpm_po in self.input_packages:
self.input_packages.add(po)
if srpm_po in self.fulltree_packages:
self.fulltree_packages.add(po)
if srpm_po in self.langpack_packages:
self.langpack_packages.add(po)
if srpm_po in self.multilib_packages:
self.multilib_packages.add(po)
added.add(po)
return added
def _downloadPackageList(self, polist, relpkgdir):
"""Cycle through the list of package objects and
download them from their respective repos."""
for pkg in sorted(polist):
repo = self.ayum.repos.getRepo(pkg.repoid)
self.logger.info("Downloading %s.%s from %s",
pkg.name, pkg.arch, repo.baseurl or repo.mirrorlist)
pkgdir = os.path.join(self.config.get('pungi', 'destdir'),
self.config.get('pungi', 'version'),
self.config.get('pungi', 'variant'),
relpkgdir)
# Ensure the pkgdir exists, force if requested, and make sure we clean it out
if relpkgdir.endswith('SRPMS'):
# Since we share source dirs with other arches don't clean, but do allow us to use it
2015-02-05 15:56:24 +00:00
pungi.util._ensuredir(pkgdir, self.logger, force=True, clean=False)
else:
2015-02-05 15:56:24 +00:00
pungi.util._ensuredir(pkgdir, self.logger, force=self.config.getboolean('pungi', 'force'), clean=True)
probs = self.ayum.downloadPkgs(polist)
if len(probs.keys()) > 0:
self.logger.error("Errors were encountered while downloading packages.")
for key in probs.keys():
errors = yum.misc.unique(probs[key])
for error in errors:
self.logger.error("%s: %s" % (key, error))
sys.exit(1)
for po in polist:
basename = os.path.basename(po.relativepath)
local = po.localPkg()
if self.config.getboolean('pungi', 'nohash'):
target = os.path.join(pkgdir, basename)
else:
target = os.path.join(pkgdir, po.name[0].lower(), basename)
# Make sure we have the hashed dir available to link into we only want dirs there to corrospond to packages
# that we are including so we can not just do A-Z 0-9
2015-02-05 15:56:24 +00:00
pungi.util._ensuredir(os.path.join(pkgdir, po.name[0].lower()), self.logger, force=True, clean=False)
# Link downloaded package in (or link package from file repo)
try:
2015-02-05 15:56:24 +00:00
pungi.util._link(local, target, self.logger, force=True)
continue
except:
self.logger.error("Unable to link %s from the yum cache." % po.name)
sys.exit(1)
self.logger.info('Finished downloading packages.')
@yumlocked
def downloadPackages(self):
"""Download the package objects obtained in getPackageObjects()."""
self._downloadPackageList(self.po_list,
2012-11-12 14:59:02 +00:00
os.path.join(self.tree_arch,
self.config.get('pungi', 'osdir'),
self.config.get('pungi', 'product_path')))
def makeCompsFile(self):
"""Gather any comps files we can from repos and merge them into one."""
ourcompspath = os.path.join(self.workdir, '%s-%s-comps.xml' % (self.config.get('pungi', 'family'), self.config.get('pungi', 'version')))
# Filter out things we don't include
ourgroups = []
for item in self.ksparser.handler.packages.groupList:
g = self.ayum.comps.return_group(item.name)
if g:
ourgroups.append(g.groupid)
allgroups = [g.groupid for g in self.ayum.comps.get_groups()]
for group in allgroups:
if group not in ourgroups and not self.ayum.comps.return_group(group).langonly:
self.logger.info('Removing extra group %s from comps file' % (group,))
del self.ayum.comps._groups[group]
groups = [g.groupid for g in self.ayum.comps.get_groups()]
envs = self.ayum.comps.get_environments()
for env in envs:
for group in env.groups:
if group not in groups:
self.logger.info('Removing incomplete environment %s from comps file' % (env,))
del self.ayum.comps._environments[env.environmentid]
break
ourcomps = open(ourcompspath, 'w')
ourcomps.write(self.ayum.comps.xml())
ourcomps.close()
# Disable this until https://bugzilla.redhat.com/show_bug.cgi?id=442097 is fixed.
# Run the xslt filter over our comps file
#compsfilter = ['/usr/bin/xsltproc', '--novalid']
#compsfilter.append('-o')
#compsfilter.append(ourcompspath)
#compsfilter.append('/usr/share/pungi/comps-cleanup.xsl')
#compsfilter.append(ourcompspath)
2015-02-05 15:56:24 +00:00
#pungi.util._doRunCommand(compsfilter, self.logger)
@yumlocked
def downloadSRPMs(self):
"""Cycle through the list of srpms and
find the package objects for them, Then download them."""
# do the downloads
self._downloadPackageList(self.srpm_po_list, os.path.join('source', 'SRPMS'))
@yumlocked
def downloadDebuginfo(self):
"""Cycle through the list of debuginfo rpms and
download them."""
# do the downloads
self._downloadPackageList(self.debuginfo_po_list, os.path.join(self.tree_arch, 'debug'))
def _list_packages(self, po_list):
"""Cycle through the list of packages and return their paths."""
result = []
for po in po_list:
if po.repoid in self.lookaside_repos:
continue
flags = []
# input
if po in self.input_packages:
flags.append("input")
# comps
if po in self.comps_packages:
flags.append("comps")
# prepopulate
if po in self.prepopulate_packages:
flags.append("prepopulate")
# langpack
if po in self.langpack_packages:
flags.append("langpack")
# multilib
if po in self.multilib_packages:
flags.append("multilib")
# fulltree
if po in self.fulltree_packages:
flags.append("fulltree")
# fulltree-exclude
if is_source(po):
srpm_name = po.name
else:
srpm_name = po.sourcerpm.rsplit("-", 2)[0]
if srpm_name in self.fulltree_excludes:
flags.append("fulltree-exclude")
result.append({
"path": os.path.join(po.basepath or "", po.relativepath),
"flags": sorted(flags),
})
result.sort(lambda x, y: cmp(x["path"], y["path"]))
return result
def list_packages(self):
"""Cycle through the list of RPMs and return their paths."""
return self._list_packages(self.po_list)
def list_srpms(self):
"""Cycle through the list of SRPMs and return their paths."""
return self._list_packages(self.srpm_po_list)
def list_debuginfo(self):
"""Cycle through the list of DEBUGINFO RPMs and return their paths."""
return self._list_packages(self.debuginfo_po_list)
def _size_packages(self, po_list):
return sum([ po.size for po in po_list if po.repoid not in self.lookaside_repos ])
def size_packages(self):
return self._size_packages(self.po_list)
def size_srpms(self):
return self._size_packages(self.srpm_po_list)
def size_debuginfo(self):
return self._size_packages(self.debuginfo_po_list)
def writeinfo(self, line):
"""Append a line to the infofile in self.infofile"""
f=open(self.infofile, "a+")
f.write(line.strip() + "\n")
f.close()
def mkrelative(self, subfile):
"""Return the relative path for 'subfile' underneath the version dir."""
basedir = os.path.join(self.destdir, self.config.get('pungi', 'version'))
if subfile.startswith(basedir):
return subfile.replace(basedir + os.path.sep, '')
def _makeMetadata(self, path, cachedir, comps=False, repoview=False, repoviewtitle=False,
baseurl=False, output=False, basedir=False, update=True,
compress_type=None):
"""Create repodata and repoview."""
# Define outputdir
if output:
outputdir = output
else:
outputdir = path
# Define revision if SOURCE_DATE_EPOCH exists in env
if 'SOURCE_DATE_EPOCH' in os.environ:
revision = os.environ['SOURCE_DATE_EPOCH']
else:
revision = None
createrepo_wrapper = CreaterepoWrapper(createrepo_c=True)
createrepo = createrepo_wrapper.get_createrepo_cmd(directory=path, update=update, outputdir=outputdir,
unique_md_filenames=True, database=True, groupfile=comps,
basedir=basedir, baseurl=baseurl, revision=revision,
compress_type=compress_type)
self.logger.info('Making repodata')
pungi.util._doRunCommand(createrepo, self.logger)
if repoview:
# setup the repoview call
repoview = ['/usr/bin/repoview']
repoview.append('--quiet')
repoview.append('--state-dir')
repoview.append(os.path.join(cachedir, 'repoviewcache'))
if repoviewtitle:
repoview.append('--title')
repoview.append(repoviewtitle)
repoview.append(path)
# run the command
2015-02-05 15:56:24 +00:00
pungi.util._doRunCommand(repoview, self.logger)
def doCreaterepo(self, comps=True):
"""Run createrepo to generate repodata in the tree."""
compsfile = None
if comps:
compsfile = os.path.join(self.workdir, '%s-%s-comps.xml' % (self.config.get('pungi', 'family'), self.config.get('pungi', 'version')))
# setup the cache dirs
for target in ['createrepocache', 'repoviewcache']:
2015-02-05 15:56:24 +00:00
pungi.util._ensuredir(os.path.join(self.config.get('pungi', 'cachedir'),
target),
self.logger,
force=True)
repoviewtitle = '%s %s - %s' % (self.config.get('pungi', 'family'),
self.config.get('pungi', 'version'),
2012-11-12 14:59:02 +00:00
self.tree_arch)
cachedir = self.config.get('pungi', 'cachedir')
compress_type = self.config.get('pungi', 'compress_type')
# setup the createrepo call
self._makeMetadata(self.topdir, cachedir, compsfile,
repoview=True, repoviewtitle=repoviewtitle,
compress_type=compress_type)
# create repodata for debuginfo
if self.config.getboolean('pungi', 'debuginfo'):
path = os.path.join(self.archdir, 'debug')
if not os.path.isdir(path):
2012-11-12 14:59:02 +00:00
self.logger.debug("No debuginfo for %s" % self.tree_arch)
return
self._makeMetadata(path, cachedir, repoview=False,
compress_type=compress_type)
def _shortenVolID(self):
"""shorten the volume id to make sure its under 32 characters"""
2014-12-09 17:51:30 +00:00
substitutions = {'Workstation': 'WS',
'Server': 'S',
'Cloud': 'C',
'Alpha': 'A',
'Beta': 'B',
'TC': 'T'}
if self.config.get('pungi', 'variant'):
name = '%s-%s' % (self.config.get('pungi', 'family'), self.config.get('pungi', 'variant'))
else:
name = self.config.get('pungi', 'family')
version = self.config.get('pungi', 'version')
arch = self.tree_arch
2014-12-09 17:51:30 +00:00
for k, v in substitutions.iteritems():
if k in name:
name = name.replace(k, v)
if k in version:
version = version.replace(k, v)
volid = "%s-%s-%s" % (name, version, arch)
if len(volid) > 32:
raise RuntimeError("Volume ID %s is longer than 32 characters" % volid)
else:
return volid
def doBuildinstall(self):
"""Run lorax on the tree."""
cmd = ["lorax"]
cmd.extend(["--workdir", self.workdir])
cmd.extend(["--logfile", os.path.join(self.config.get('pungi', 'destdir'), 'logs/lorax-%s.log' % (self.config.get('pungi', 'arch')))])
try:
# Convert url method to a repo
self.ksparser.handler.repo.methodToRepo()
except:
pass
for repo in self.ksparser.handler.repo.repoList:
if repo.mirrorlist:
# The not bool() thing is because pykickstart is yes/no on
# whether to ignore groups, but yum is a yes/no on whether to
# include groups. Awkward.
repo.mirrorlist = yum.parser.varReplace(repo.mirrorlist, self.ayum.conf.yumvar)
cmd.extend(["--mirrorlist", repo.mirrorlist])
else:
repo.baseurl = yum.parser.varReplace(repo.baseurl, self.ayum.conf.yumvar)
cmd.extend(["--source", repo.baseurl])
# Add the repo in the destdir to our yum object
cmd.extend(["--source", "file://%s" % self.topdir])
cmd.extend(["--product", self.config.get('pungi', 'family')])
cmd.extend(["--version", self.config.get('pungi', 'version')])
cmd.extend(["--release", "%s %s" % (self.config.get('pungi', 'family'), self.config.get('pungi', 'version'))])
if self.config.get('pungi', 'variant'):
cmd.extend(["--variant", self.config.get('pungi', 'variant')])
cmd.extend(["--bugurl", self.config.get('pungi', 'bugurl')])
if self.config.getboolean('pungi', 'isfinal'):
cmd.append("--isfinal")
cmd.extend(["--volid", self._shortenVolID()])
# on ppc64 we need to tell lorax to only use ppc64 packages so that the media will run on all 64 bit ppc boxes
2013-08-20 16:33:44 +00:00
if self.tree_arch == 'ppc64':
cmd.extend(["--buildarch", "ppc64"])
elif self.tree_arch == 'ppc64le':
cmd.extend(["--buildarch", "ppc64le"])
# Only supported mac hardware is x86 make sure we only enable mac support on arches that need it
if self.tree_arch in ['x86_64'] and not self.is_nomacboot:
cmd.append("--macboot")
else:
cmd.append("--nomacboot")
try:
cmd.extend(["--conf", self.config.get('lorax', 'conf_file')])
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
pass
try:
cmd.extend(["--installpkgs", self.config.get('lorax', 'installpkgs')])
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
pass
if self.rootfs_size != "False":
cmd.extend(["--rootfs-size", self.rootfs_size])
# Allow the output directory to exist.
cmd.append("--force")
# MUST be last in the list
cmd.append(self.topdir)
self.logger.info(" ".join(cmd))
2015-03-12 21:12:38 +00:00
pungi.util._doRunCommand(cmd, self.logger)
# write out the tree data for snake
self.writeinfo('tree: %s' % self.mkrelative(self.topdir))
# Write out checksums for verifytree
# First open the treeinfo file so that we can config parse it
treeinfofile = os.path.join(self.topdir, '.treeinfo')
try:
treefile = open(treeinfofile, 'r')
except IOError:
self.logger.error("Could not read .treeinfo file: %s" % treefile)
sys.exit(1)
# Create a ConfigParser object out of the contents so that we can
# write it back out later and not worry about formatting
treeinfo = SortedConfigParser()
treeinfo.readfp(treefile)
treefile.close()
treeinfo.add_section('checksums')
# Create a function to use with os.path.walk to sum the files
# basepath is used to make the sum output relative
sums = []
def getsum(basepath, dir, files):
for file in files:
path = os.path.join(dir, file)
# don't bother summing directories. Won't work.
if os.path.isdir(path):
continue
2015-02-05 15:56:24 +00:00
sum = pungi.util._doCheckSum(path, 'sha256', self.logger)
outpath = path.replace(basepath, '')
sums.append((outpath, sum))
# Walk the os/images path to get sums of all the files
os.path.walk(os.path.join(self.topdir, 'images'), getsum, self.topdir + '/')
# Capture PPC images
if self.tree_arch in ['ppc', 'ppc64', 'ppc64le']:
os.path.walk(os.path.join(self.topdir, 'ppc'), getsum, self.topdir + '/')
# Get a checksum of repomd.xml since it has within it sums for other files
repomd = os.path.join(self.topdir, 'repodata', 'repomd.xml')
2015-02-05 15:56:24 +00:00
sum = pungi.util._doCheckSum(repomd, 'sha256', self.logger)
sums.append((os.path.join('repodata', 'repomd.xml'), sum))
# Now add the sums, and write the config out
try:
treefile = open(treeinfofile, 'w')
except IOError:
self.logger.error("Could not open .treeinfo for writing: %s" % treefile)
sys.exit(1)
for path, sum in sums:
treeinfo.set('checksums', path, sum)
# Extract name of kernel images
pr = re.compile('images-(.*)')
images = []
for img in treeinfo.sections():
if pr.match(img):
images.append(pr.match(img).group(1))
# Extract information from pre-productmd treeinfos 'general' section
name = treeinfo.get('general', 'family')
version = treeinfo.get('general', 'version')
arch = treeinfo.get('general', 'arch')
platforms = ','.join(images)
timestamp = int(float(treeinfo.get('general', 'timestamp')))
# Set/modify 'general' section
treeinfo.set('general', 'variant', name)
treeinfo.set('general', 'timestamp', timestamp)
treeinfo.set('general', 'packagedir', 'Packages')
treeinfo.set('general', 'repository', '.')
treeinfo.set('general', 'platforms', platforms)
# Add 'header' section
treeinfo.add_section('header')
treeinfo.set('header', 'version', '1.0')
# Add 'release' section
treeinfo.add_section('release')
treeinfo.set('release', 'name', name)
treeinfo.set('release', 'short', name)
treeinfo.set('release', 'version', version)
# Add 'tree' section
treeinfo.add_section('tree')
treeinfo.set('tree', 'arch', arch)
treeinfo.set('tree', 'build_timestamp', timestamp)
treeinfo.set('tree', 'platforms', platforms)
treeinfo.set('tree', 'variants', name)
# Add 'variant-VARIANTNAME' section
variant_section_name = 'variant-' + name
treeinfo.add_section(variant_section_name)
treeinfo.set(variant_section_name, 'id', name)
treeinfo.set(variant_section_name, 'name', name)
treeinfo.set(variant_section_name, 'packages', 'Packages')
treeinfo.set(variant_section_name, 'repository', '.')
treeinfo.set(variant_section_name, 'type', 'variant')
treeinfo.set(variant_section_name, 'uid', name)
treeinfo.write(treefile)
treefile.close()
def doGetRelnotes(self):
"""Get extra files from packages in the tree to put in the topdir of
the tree."""
docsdir = os.path.join(self.workdir, 'docs')
relnoterpms = self.config.get('pungi', 'relnotepkgs').split()
fileres = []
for pattern in self.config.get('pungi', 'relnotefilere').split():
fileres.append(re.compile(pattern))
dirres = []
for pattern in self.config.get('pungi', 'relnotedirre').split():
dirres.append(re.compile(pattern))
2015-02-05 15:56:24 +00:00
pungi.util._ensuredir(docsdir, self.logger, force=self.config.getboolean('pungi', 'force'), clean=True)
# Expload the packages we list as relnote packages
pkgs = os.listdir(os.path.join(self.topdir, self.config.get('pungi', 'product_path')))
rpm2cpio = ['/usr/bin/rpm2cpio']
cpio = ['cpio', '-imud']
for pkg in pkgs:
pkgname = pkg.rsplit('-', 2)[0]
for relnoterpm in relnoterpms:
if pkgname == relnoterpm:
extraargs = [os.path.join(self.topdir, self.config.get('pungi', 'product_path'), pkg)]
try:
p1 = subprocess.Popen(rpm2cpio + extraargs, cwd=docsdir, stdout=subprocess.PIPE)
(out, err) = subprocess.Popen(cpio, cwd=docsdir, stdin=p1.stdout, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, universal_newlines=True).communicate()
except:
self.logger.error("Got an error from rpm2cpio")
self.logger.error(err)
raise
if out:
self.logger.debug(out)
# Walk the tree for our files
for dirpath, dirname, filelist in os.walk(docsdir):
for filename in filelist:
for regex in fileres:
if regex.match(filename) and not os.path.exists(os.path.join(self.topdir, filename)):
self.logger.info("Linking release note file %s" % filename)
2015-02-05 15:56:24 +00:00
pungi.util._link(os.path.join(dirpath, filename),
os.path.join(self.topdir, filename),
self.logger,
force=self.config.getboolean('pungi',
'force'))
self.common_files.append(filename)
# Walk the tree for our dirs
for dirpath, dirname, filelist in os.walk(docsdir):
for directory in dirname:
for regex in dirres:
if regex.match(directory) and not os.path.exists(os.path.join(self.topdir, directory)):
self.logger.info("Copying release note dir %s" % directory)
shutil.copytree(os.path.join(dirpath, directory), os.path.join(self.topdir, directory))
2009-02-10 22:49:59 +00:00
def _doIsoChecksum(self, path, csumfile):
"""Simple function to wrap creating checksums of iso files."""
try:
2009-02-10 22:49:59 +00:00
checkfile = open(csumfile, 'a')
except IOError:
2009-02-10 22:49:59 +00:00
self.logger.error("Could not open checksum file: %s" % csumfile)
2009-02-10 22:49:59 +00:00
self.logger.info("Generating checksum of %s" % path)
2015-02-05 15:56:24 +00:00
checksum = pungi.util._doCheckSum(path, 'sha256', self.logger)
2009-02-10 22:49:59 +00:00
if checksum:
checkfile.write("SHA256 (%s) = %s\n" % (os.path.basename(path), checksum.replace('sha256:', '')))
else:
2009-02-10 22:49:59 +00:00
self.logger.error('Failed to generate checksum for %s' % checkfile)
sys.exit(1)
2009-02-10 22:49:59 +00:00
checkfile.close()
2010-11-12 17:27:10 +00:00
def doCreateIsos(self):
"""Create iso of the tree."""
2013-08-20 16:33:44 +00:00
if self.tree_arch.startswith('arm'):
self.logger.info("ARCH: arm, not doing doCreateIsos().")
return
ppcbootinfo = '/usr/share/lorax/config_files/ppc'
2015-02-05 15:56:24 +00:00
pungi.util._ensuredir(self.isodir, self.logger,
force=self.config.getboolean('pungi', 'force'),
clean=True) # This is risky...
# setup the base command
mkisofs = ['/usr/bin/xorriso', '-as', 'mkisofs']
mkisofs.extend(['-v', '-U', '-J', '--joliet-long', '-R', '-T', '-m', 'repoview', '-m', 'boot.iso']) # common mkisofs flags
x86bootargs = ['-b', 'isolinux/isolinux.bin', '-c', 'isolinux/boot.cat',
'-no-emul-boot', '-boot-load-size', '4', '-boot-info-table']
efibootargs = ['-eltorito-alt-boot', '-e', 'images/efiboot.img',
'-no-emul-boot']
macbootargs = ['-eltorito-alt-boot', '-e', 'images/macboot.img',
'-no-emul-boot']
ia64bootargs = ['-b', 'images/boot.img', '-no-emul-boot']
ppcbootargs = ['-part', '-hfs', '-r', '-l', '-sysid', 'PPC', '-no-desktop', '-allow-multidot', '-chrp-boot']
ppcbootargs.append('-map')
ppcbootargs.append(os.path.join(ppcbootinfo, 'mapping'))
ppcbootargs.append('-hfs-bless') # must be last
isohybrid = ['/usr/bin/isohybrid']
isohybrid.extend(['--id', '42'])
# Check the size of the tree
# This size checking method may be bunk, accepting patches...
2012-11-12 14:59:02 +00:00
if not self.tree_arch == 'source':
treesize = int(subprocess.Popen(mkisofs + ['-print-size', '-quiet', self.topdir], stdout=subprocess.PIPE).communicate()[0])
else:
srcdir = os.path.join(self.config.get('pungi', 'destdir'), self.config.get('pungi', 'version'),
self.config.get('pungi', 'variant'), 'source', 'SRPMS')
treesize = int(subprocess.Popen(mkisofs + ['-print-size', '-quiet', srcdir], stdout=subprocess.PIPE).communicate()[0])
# Size returned is 2KiB clusters or some such. This translates that to MiB.
treesize = treesize * 2048 / 1024 / 1024
if treesize > 700: # we're larger than a 700meg CD
isoname = '%s-DVD-%s-%s.iso' % (self.config.get('pungi', 'iso_basename'), self.tree_arch,
self.config.get('pungi', 'version'))
else:
isoname = '%s-%s-%s.iso' % (self.config.get('pungi', 'iso_basename'), self.tree_arch,
self.config.get('pungi', 'version'))
isofile = os.path.join(self.isodir, isoname)
# setup the extra mkisofs args
extraargs = []
2012-11-12 14:59:02 +00:00
if self.tree_arch == 'i386' or self.tree_arch == 'x86_64':
extraargs.extend(x86bootargs)
2012-11-12 14:59:02 +00:00
if self.tree_arch == 'x86_64':
extraargs.extend(efibootargs)
isohybrid.append('-u')
if (not self.is_nomacboot) and os.path.exists(os.path.join(self.topdir, 'images', 'macboot.img')):
extraargs.extend(macbootargs)
isohybrid.append('-m')
2012-11-12 14:59:02 +00:00
elif self.tree_arch == 'ia64':
extraargs.extend(ia64bootargs)
2012-11-12 14:59:02 +00:00
elif self.tree_arch.startswith('ppc'):
extraargs.extend(ppcbootargs)
extraargs.append(os.path.join(self.topdir, "ppc/mac"))
elif self.tree_arch.startswith('aarch64'):
extraargs.extend(efibootargs)
# NOTE: if this doesn't match what's in the bootloader config, the
# image won't be bootable!
extraargs.append('-V')
extraargs.append(self._shortenVolID())
extraargs.extend(['-o', isofile])
isohybrid.append(isofile)
2012-11-12 14:59:02 +00:00
if not self.tree_arch == 'source':
extraargs.append(self.topdir)
else:
extraargs.append(os.path.join(self.archdir, 'SRPMS'))
if self.config.get('pungi', 'no_dvd') == "False":
# run the command
2015-02-05 15:56:24 +00:00
pungi.util._doRunCommand(mkisofs + extraargs, self.logger)
# Run isohybrid on the iso as long as its not the source iso
if os.path.exists("/usr/bin/isohybrid") and not self.tree_arch == 'source':
2015-02-05 15:56:24 +00:00
pungi.util._doRunCommand(isohybrid, self.logger)
2011-05-26 15:01:43 +00:00
# implant md5 for mediacheck on all but source arches
if not self.tree_arch == 'source':
2015-02-05 15:56:24 +00:00
pungi.util._doRunCommand(['/usr/bin/implantisomd5', isofile], self.logger)
# shove the checksum into a file
csumfile = os.path.join(self.isodir, '%s-%s-%s-CHECKSUM' % (
self.config.get('pungi', 'iso_basename'),
self.config.get('pungi', 'version'),
2012-11-12 14:59:02 +00:00
self.tree_arch))
# Write a line about what checksums are used.
# sha256sum is magic...
file = open(csumfile, 'w')
file.write('# The image checksum(s) are generated with sha256sum.\n')
file.close()
if self.config.get('pungi', 'no_dvd') == "False":
self._doIsoChecksum(isofile, csumfile)
# Write out a line describing the media
self.writeinfo('media: %s' % self.mkrelative(isofile))
# Now link the boot iso
2012-11-12 14:59:02 +00:00
if not self.tree_arch == 'source' and \
os.path.exists(os.path.join(self.topdir, 'images', 'boot.iso')):
isoname = '%s-netinst-%s-%s.iso' % (self.config.get('pungi', 'iso_basename'),
self.tree_arch, self.config.get('pungi', 'version'))
isofile = os.path.join(self.isodir, isoname)
# link the boot iso to the iso dir
2015-02-05 15:56:24 +00:00
pungi.util._link(os.path.join(self.topdir, 'images', 'boot.iso'), isofile, self.logger)
# shove the checksum into a file
self._doIsoChecksum(isofile, csumfile)
self.logger.info("CreateIsos is done.")