almalinux
/
pungi
15
1
Fork 0
Code Issues Pull Requests Releases Activity

Add script to generate unified ISOs 

This a standalone script that will look into a compose and create
unified ISO for each architecture. The ISO contains RPM repositories for
all variants that have the arch.

Known issues:
 * The filename does not respect settings. This is tricky because the
   name could include variant name, which we don't have here (by design
   of unified ISO).
 * The same is true for volume id.

In order to test the feature without running actual compose, we need to
add essentially a big chunk of compose. Most of the files are empty, as
their content is never accessed.

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2016-10-12 15:42:22 +02:00
parent 2d404c88e6
commit e260fe5581
58 changed files with 1720 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
MANIFEST.in
View File

@ -10,4 +10,4 @@ include doc/*
include tests/*
include tests/data/*
include tests/data/*/*
recursive-include tests/fixtures *.json *.xml *.bz2 *.gz
recursive-include tests/fixtures *.json *.xml *.bz2 *.gz *.iso *.log MD5SUM SHA1SUM SHA256SUM treeinfo

42
bin/pungi-create-unified-isos Executable file
View File

@ -0,0 +1,42 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
This script creates unified ISOs for a specified compose.
Unified ISOs are created per architecture and contain all variant packages and
repos.
"""
import argparse
import os
import sys
here = sys.path[0]
if here != '/usr/bin':
# Git checkout
sys.path[0] = os.path.dirname(here)
from pungi_utils.unified_isos import UnifiedISO
def parse_args():
parser = argparse.ArgumentParser(add_help=True)
parser.add_argument(
'compose',
metavar='<compose-path>',
nargs=1,
help='path to compose',
)
return parser.parse_args()
def main():
args = parse_args()
iso = UnifiedISO(args.compose[0])
iso.create(delete_temp=True)
if __name__ == '__main__':
main()

76
pungi/phases/image_checksum.py
View File

@ -68,36 +68,42 @@ class ImageChecksumPhase(PhaseBase):
return base_checksum_name
def run(self):
for (variant, arch, path), images in self._get_images().iteritems():
checksums = {}
topdir = self.compose.paths.compose.topdir()
for (variant, arch, path), images in get_images(topdir, self.compose.im).iteritems():
base_checksum_name = self._get_base_filename(variant, arch)
for image in images:
filename = os.path.basename(image.path)
full_path = os.path.join(path, filename)
if not os.path.exists(full_path):
continue
make_checksums(variant, arch, path, images,
self.checksums, base_checksum_name, self.one_file)
digests = shortcuts.compute_file_checksums(full_path, self.checksums)
for checksum, digest in digests.iteritems():
checksums.setdefault(checksum, {})[filename] = digest
image.add_checksum(None, checksum, digest)
if not self.one_file:
dump_checksums(path, checksum,
{filename: digest},
'%s.%sSUM' % (filename, checksum.upper()))
if not checksums:
continue
def make_checksums(variant, arch, path, images, checksum_types, base_checksum_name, one_file):
checksums = {}
for image in images:
filename = os.path.basename(image.path)
full_path = os.path.join(path, filename)
if not os.path.exists(full_path):
continue
if self.one_file:
dump_checksums(path, self.checksums[0],
checksums[self.checksums[0]],
base_checksum_name + 'CHECKSUM')
else:
for checksum in self.checksums:
dump_checksums(path, checksum,
checksums[checksum],
'%s%sSUM' % (base_checksum_name, checksum.upper()))
digests = shortcuts.compute_file_checksums(full_path, checksum_types)
for checksum, digest in digests.iteritems():
checksums.setdefault(checksum, {})[filename] = digest
image.add_checksum(None, checksum, digest)
if not one_file:
dump_checksums(path, checksum,
{filename: digest},
'%s.%sSUM' % (filename, checksum.upper()))
if not checksums:
return
if one_file:
dump_checksums(path, checksum_types[0],
checksums[checksum_types[0]],
base_checksum_name + 'CHECKSUM')
else:
for checksum in checksums:
dump_checksums(path, checksum,
checksums[checksum],
'%s%sSUM' % (base_checksum_name, checksum.upper()))
def dump_checksums(dir, alg, checksums, filename):
@ -108,6 +114,22 @@ def dump_checksums(dir, alg, checksums, filename):
:param checksums: mapping from filenames to checksums
:param filename: what to call the file
"""
with open(os.path.join(dir, filename), 'w') as f:
checksum_file = os.path.join(dir, filename)
with open(checksum_file, 'w') as f:
for file, checksum in checksums.iteritems():
f.write('%s (%s) = %s\n' % (alg.upper(), file, checksum))
return checksum_file
def get_images(top_dir, manifest):
"""Returns a mapping from directories to sets of ``Image``s.
The paths to dirs are absolute.
"""
images = {}
for variant in manifest.images:
for arch in manifest.images[variant]:
for image in manifest.images[variant][arch]:
path = os.path.dirname(os.path.join(top_dir, image.path))
images.setdefault((variant, arch, path), []).append(image)
return images

0
pungi_utils/__init__.py Normal file
View File

380
pungi_utils/unified_isos.py Normal file
View File

@ -0,0 +1,380 @@
# -*- coding: utf-8 -*-
"""
This script creates unified ISOs for a specified compose.
Unified ISOs are created per architecture and
contain all variant packages and repos.
TODO:
* jigdo
"""
from __future__ import print_function
import copy
import errno
import glob
import json
import os
import shutil
import sys
import tempfile
import productmd
import productmd.compose
import productmd.images
import productmd.treeinfo
from kobo.shortcuts import run, compute_file_checksums
import pungi.linker
import pungi.wrappers.createrepo
from pungi.util import makedirs
from pungi.compose_metadata.discinfo import write_discinfo as create_discinfo
from pungi.wrappers import iso
from pungi.phases.image_checksum import dump_checksums, get_images, make_checksums
def ti_merge(one, two):
assert one.tree.arch == two.tree.arch
for variant in two.variants.get_variants(recursive=False):
if variant.uid in one.variants:
continue
var = productmd.treeinfo.Variant(one)
var.id = variant.id
var.uid = variant.uid
var.name = variant.name
var.type = variant.type
for i in ("debug_packages", "debug_repository", "packages", "repository",
"source_packages", "source_repository"):
setattr(var, i, getattr(variant, i, None))
one.variants.add(var)
class UnifiedISO(object):
def __init__(self, compose_path, output_path=None):
self.compose_path = os.path.abspath(compose_path)
compose_subdir = os.path.join(self.compose_path, "compose")
if os.path.exists(compose_subdir):
self.compose_path = compose_subdir
self.compose = productmd.compose.Compose(compose_path)
self.ci = self.compose.info
self.linker = pungi.linker.Linker()
temp_topdir = os.path.abspath(os.path.join(self.compose_path, "..", "work"))
self.temp_dir = tempfile.mkdtemp(prefix="unified_isos_", dir=temp_topdir)
self.treeinfo = {} # {arch/src: TreeInfo}
self.repos = {} # {arch/src: {variant: new_path}
self.comps = {} # {arch/src: {variant: old_path}
self.productid = {} # {arch/stc: {variant: old_path}
self.images = {} # {arch/src: [*.iso, *.iso.{md5,sha1,sha256}sum]}
self.conf = self.read_config()
def create(self, delete_temp=True):
print("Creating unified ISOs for: {0}".format(self.compose_path))
try:
self.link_to_temp()
self.createrepo()
self.discinfo()
self.createiso()
self.link_to_compose()
self.update_checksums()
finally:
if delete_temp:
shutil.rmtree(self.temp_dir)
def _link_tree(self, dir, variant, arch):
blacklist_files = [".treeinfo", ".discinfo", "boot.iso", "media.repo"]
blacklist_dirs = ["repodata"]
for root, dirs, files in os.walk(dir):
for i in blacklist_dirs:
if i in dirs:
dirs.remove(i)
for fn in files:
if fn in blacklist_files:
continue
old_path = os.path.join(root, fn)
if fn.endswith(".rpm"):
new_path = os.path.join(self.temp_dir, "trees", arch, variant.uid, fn)
self.repos.setdefault(arch, {})[variant.uid] = os.path.dirname(new_path)
else:
old_relpath = os.path.relpath(old_path, dir)
new_path = os.path.join(self.temp_dir, "trees", arch, old_relpath)
makedirs(os.path.dirname(new_path))
self.linker.link(old_path, new_path)
def link_to_temp(self):
# copy files to new location; change RPM location to $variant_uid
for variant in self.ci.get_variants(recursive=False):
for arch in variant.arches:
print("Processing: {0}.{1}".format(variant.uid, arch))
tree_dir = os.path.join(self.compose_path, variant.paths.os_tree[arch])
ti = productmd.treeinfo.TreeInfo()
try:
ti.load(os.path.join(tree_dir, ".treeinfo"))
except IOError as exc:
if exc.errno != errno.ENOENT:
raise
print('Tree %s.%s has no .treeinfo, skipping...'
% (variant.uid, arch),
file=sys.stderr)
continue
arch_ti = self.treeinfo.get(arch)
if arch_ti is None:
arch_ti = ti
self.treeinfo[arch] = arch_ti
else:
ti_merge(arch_ti, ti)
if arch_ti.tree.arch != arch:
raise RuntimeError('Treeinfo arch mismatch')
# override paths
arch_ti[variant.uid].repository = variant.uid
arch_ti[variant.uid].packages = variant.uid
comps_path = glob.glob(os.path.join(self.compose_path,
variant.paths.repository[arch],
"repodata", "*comps*.xml"))
if comps_path:
self.comps.setdefault(arch, {})[variant.uid] = comps_path[0]
productid_path = os.path.join(self.compose_path, variant.paths.repository[arch],
"repodata", "productid")
self.productid.setdefault(arch, {})[variant.uid] = productid_path
self._link_tree(tree_dir, variant, arch)
# sources
print("Processing: {0}.{1}".format(variant.uid, "src"))
tree_dir = os.path.join(self.compose_path, variant.paths.source_tree[arch])
ti = productmd.treeinfo.TreeInfo()
ti.load(os.path.join(tree_dir, ".treeinfo"))
arch_ti = self.treeinfo.get("src")
if arch_ti is None:
arch_ti = ti
self.treeinfo["src"] = arch_ti
else:
ti_merge(arch_ti, ti)
if arch_ti.tree.arch != "src":
raise RuntimeError('Treeinfo arch mismatch')
# override paths
arch_ti[variant.uid].repository = variant.uid
arch_ti[variant.uid].packages = variant.uid
# set to None, replace with source_*; requires productmd changes or upstream version
# arch_ti[variant.uid].source_repository = variant.uid
# arch_ti[variant.uid].source_packages = variant.uid
self._link_tree(tree_dir, variant, 'src')
def createrepo(self):
# remove old repomd.xml checksums from treeinfo
for arch, ti in self.treeinfo.iteritems():
print("Removing old repomd.xml checksums from treeinfo: {0}".format(arch))
for i in ti.checksums.checksums.keys():
if "repomd.xml" in i:
del ti.checksums.checksums[i]
# write new per-variant repodata
cr = pungi.wrappers.createrepo.CreaterepoWrapper(createrepo_c=True)
for arch in self.repos:
ti = self.treeinfo[arch]
for variant in self.repos[arch]:
print("Creating repodata: {0}.{1}".format(variant, arch))
tree_dir = os.path.join(self.temp_dir, "trees", arch)
repo_path = self.repos[arch][variant]
comps_path = self.comps.get(arch, {}).get(variant, None)
cmd = cr.get_createrepo_cmd(repo_path, groupfile=comps_path, update=True)
run(cmd, show_cmd=True)
productid_path = self.productid.get(arch, {}).get(variant, None)
if productid_path:
print("Adding productid to repodata: {0}.{1}".format(variant, arch))
repo_dir = os.path.join(self.repos[arch][variant], "repodata")
new_path = os.path.join(repo_dir, os.path.basename(productid_path))
if os.path.exists(productid_path):
shutil.copy2(productid_path, new_path)
cmd = cr.get_modifyrepo_cmd(repo_dir, new_path, compress_type="gz")
run(cmd)
else:
print("WARNING: productid not found in {0}.{1}".format(variant, arch))
print("Inserting new repomd.xml checksum to treeinfo: {0}.{1}".format(variant, arch))
# insert new repomd.xml checksum to treeinfo
repomd_path = os.path.join(repo_path, "repodata", "repomd.xml")
ti.checksums.add(os.path.relpath(repomd_path, tree_dir), 'sha256', root_dir=tree_dir)
# write treeinfo
for arch, ti in self.treeinfo.iteritems():
print("Writing treeinfo: {0}".format(arch))
ti_path = os.path.join(self.temp_dir, "trees", arch, ".treeinfo")
ti.dump(ti_path)
def discinfo(self):
# write discinfo and media repo
for arch, ti in self.treeinfo.iteritems():
di_path = os.path.join(self.temp_dir, "trees", arch, ".discinfo")
description = "%s %s" % (ti.release.name, ti.release.version)
if ti.release.is_layered:
description += " for %s %s" % (ti.base_product.name, ti.base_product.version)
create_discinfo(di_path, description, arch)
def read_config(self):
try:
conf_dump = glob.glob(os.path.join(self.compose_path,
'../logs/global/config-dump*.global.log'))[0]
except IndexError:
raise RuntimeError('Config dump not found, can not generate checksums...')
with open(conf_dump) as f:
return json.load(f)
def createiso(self):
# create ISOs
im = self.compose.images
for arch, ti in self.treeinfo.items():
source_dir = os.path.join(self.temp_dir, "trees", arch)
# XXX: HARDCODED
disc_type = "dvd"
iso_arch = arch
if arch == "src":
iso_arch = "source"
iso_name = "%s-%s-%s.iso" % (self.ci.compose.id, iso_arch, disc_type)
iso_dir = os.path.join(self.temp_dir, "iso", iso_arch)
iso_path = os.path.join(iso_dir, iso_name)
print("Creating ISO for {0}: {1}".format(arch, iso_name))
makedirs(iso_dir)
volid = "%s %s %s" % (ti.release.short, ti.release.version, arch)
# create ISO
run(iso.get_mkisofs_cmd(iso_path, [source_dir], volid=volid, exclude=["./lost+found"]))
# implant MD5
supported = True
run(iso.get_implantisomd5_cmd(iso_path, supported))
checksums = compute_file_checksums(iso_path, self.conf['media_checksums'])
# write manifest file
run(iso.get_manifest_cmd(iso_path))
img = productmd.images.Image(im)
# temporary path, just a file name; to be replaced with variant specific path
img.path = os.path.basename(iso_path)
img.mtime = int(os.stat(iso_path).st_mtime)
img.size = os.path.getsize(iso_path)
img.arch = arch
# XXX: HARDCODED
img.type = "dvd"
img.format = "iso"
img.disc_number = 1
img.disc_count = 1
img.bootable = False
img.unified = True
self.images.setdefault(arch, set()).add(iso_path)
self.images.setdefault(arch, set()).add(iso_path + ".manifest")
for checksum_type, checksum in checksums.iteritems():
if not self.conf['media_checksum_one_file']:
checksum_path = dump_checksums(iso_dir, checksum_type,
{iso_name: checksum},
'%s.%sSUM' % (iso_name, checksum_type.upper()))
self.images.setdefault(arch, set()).add(checksum_path)
img.add_checksum(self.compose_path, checksum_type=checksum_type, checksum_value=checksum)
img.implant_md5 = iso.get_implanted_md5(iso_path)
try:
img.volume_id = iso.get_volume_id(iso_path)
except RuntimeError:
pass
if arch == "src":
all_arches = [i for i in self.treeinfo if i != "src"]
else:
all_arches = [arch]
for tree_arch in all_arches:
ti = self.treeinfo[tree_arch]
for variant_uid in ti.variants:
variant = ti.variants[variant_uid]
# We don't want to copy the manifest.
img.parent = None
variant_img = copy.deepcopy(img)
variant_img.parent = im
variant_img.subvariant = variant.id
paths_attr = 'isos' if arch != 'src' else 'source_isos'
paths = getattr(self.ci.variants[variant.uid].paths, paths_attr)
variant_img.path = os.path.join(
paths.get(tree_arch, os.path.join(variant.uid, tree_arch, "iso")),
os.path.basename(img.path)
)
im.add(variant.uid, tree_arch, variant_img)
im.dump(os.path.join(self.compose_path, 'metadata', 'images.json'))
def link_to_compose(self):
for variant in self.ci.get_variants(recursive=False):
for arch in variant.arches | set(['src']):
if arch == 'src':
path_type, dir = 'source_isos', 'source'
else:
path_type, dir = 'isos', arch
default_path = os.path.join(variant.uid, dir, "iso")
isos = os.path.join(self.compose_path,
getattr(variant.paths, path_type).get(arch, default_path))
makedirs(isos)
for image in self.images.get(arch, []):
dst = os.path.join(isos, os.path.basename(image))
print("Linking {0} -> {1}".format(image, dst))
self.linker.link(image, dst)
def _get_base_filename(self, variant, arch):
substs = {
'compose_id': self.compose.info.compose.id,
'release_short': self.compose.info.release.short,
'version': self.compose.info.release.version,
'date': self.compose.info.compose.date,
'respin': self.compose.info.compose.respin,
'type': self.compose.info.compose.type,
'type_suffix': self.compose.info.compose.type_suffix,
'label': self.compose.info.compose.label,
'label_major_version': self.compose.info.compose.label_major_version,
'variant': variant,
'arch': arch,
}
base_name = self.conf['media_checksum_base_filename']
if base_name:
base_name = (base_name % substs).format(**substs)
base_name += '-'
return base_name
def update_checksums(self):
for (variant, arch, path), images in get_images(self.compose_path, self.compose.images).iteritems():
base_checksum_name = self._get_base_filename(variant, arch)
make_checksums(variant, arch, path, images,
self.conf['media_checksums'],
base_checksum_name,
self.conf['media_checksum_one_file'])

0
tests/fixtures/DP-1.0-20161013.t.4/compose/Client/i386/iso/DP-1.0-20161013.t.4-Client-i386-dvd1.iso vendored Normal file
View File

1
tests/fixtures/DP-1.0-20161013.t.4/compose/Client/i386/iso/MD5SUM vendored Normal file
View File

@ -0,0 +1 @@
MD5 (DP-1.0-20161013.t.4-Client-i386-dvd1.iso) = 7d1e43fdb5e921a8c5d4841d1ec69f03

1
tests/fixtures/DP-1.0-20161013.t.4/compose/Client/i386/iso/SHA1SUM vendored Normal file
View File

@ -0,0 +1 @@
SHA1 (DP-1.0-20161013.t.4-Client-i386-dvd1.iso) = e9f5fd1b38febf0b8cdeafe1a4c9beb94f985c0d

1
tests/fixtures/DP-1.0-20161013.t.4/compose/Client/i386/iso/SHA256SUM vendored Normal file
View File

@ -0,0 +1 @@
SHA256 (DP-1.0-20161013.t.4-Client-i386-dvd1.iso) = 1157a2fbc0c959d6bf2ea1223cd4b097659212d0f14e621fe2801ab06c4f8413

33
tests/fixtures/DP-1.0-20161013.t.4/compose/Client/i386/os/.treeinfo vendored Normal file
View File

@ -0,0 +1,33 @@
[general]
; WARNING.0 = This section provides compatibility with pre-productmd treeinfos.
; WARNING.1 = Read productmd documentation for details about new format.
arch = i386
family = Dummy Product
name = Dummy Product 1.0
platforms = i386
timestamp = 1476343975
variant = Client
variants = Client
version = 1.0
[header]
type = productmd.treeinfo
version = 1.2
[release]
name = Dummy Product
short = DP
version = 1.0
[tree]
arch = i386
build_timestamp = 1476343975
platforms = i386
variants = Client
[variant-Client]
id = Client
name = Client
type = variant
uid = Client

0
tests/fixtures/DP-1.0-20161013.t.4/compose/Client/i386/os/Packages/d/dummy-bash-4.2.37-6.i686.rpm vendored Normal file
View File

0
tests/fixtures/DP-1.0-20161013.t.4/compose/Client/i386/os/repodata/comps-Client.i386.xml vendored Normal file
View File

0
tests/fixtures/DP-1.0-20161013.t.4/compose/Client/i386/os/repodata/repomd.xml vendored Normal file
View File

0
tests/fixtures/DP-1.0-20161013.t.4/compose/Client/source/iso/DP-1.0-20161013.t.4-Client-source-dvd1.iso vendored Normal file
View File

1
tests/fixtures/DP-1.0-20161013.t.4/compose/Client/source/iso/MD5SUM vendored Normal file
View File

@ -0,0 +1 @@
MD5 (DP-1.0-20161013.t.4-Client-source-dvd1.iso) = c61c9adc0c553dc8c65a1af0fc822952

1
tests/fixtures/DP-1.0-20161013.t.4/compose/Client/source/iso/SHA1SUM vendored Normal file
View File

@ -0,0 +1 @@
SHA1 (DP-1.0-20161013.t.4-Client-source-dvd1.iso) = 9ad372897848ee7d89f483d9db5b6412a6cafa85

1
tests/fixtures/DP-1.0-20161013.t.4/compose/Client/source/iso/SHA256SUM vendored Normal file
View File

@ -0,0 +1 @@
SHA256 (DP-1.0-20161013.t.4-Client-source-dvd1.iso) = 4993325bec4cb83fe83feaf2a66f9dfaa88b95b7a66491a6f26c104e69885586

33
tests/fixtures/DP-1.0-20161013.t.4/compose/Client/source/tree/.treeinfo vendored Normal file
View File

@ -0,0 +1,33 @@
[general]
; WARNING.0 = This section provides compatibility with pre-productmd treeinfos.
; WARNING.1 = Read productmd documentation for details about new format.
arch = src
family = Dummy Product
name = Dummy Product 1.0
platforms = src
timestamp = 1476343975
variant = Client
variants = Client
version = 1.0
[header]
type = productmd.treeinfo
version = 1.2
[release]
name = Dummy Product
short = DP
version = 1.0
[tree]
arch = src
build_timestamp = 1476343975
platforms = src
variants = Client
[variant-Client]
id = Client
name = Client
type = variant
uid = Client

0
tests/fixtures/DP-1.0-20161013.t.4/compose/Client/source/tree/Packages/d/dummy-bash-4.2.37-6.src.rpm vendored Normal file
View File

0
tests/fixtures/DP-1.0-20161013.t.4/compose/Client/source/tree/repodata/repomd.xml vendored Normal file
View File

0
tests/fixtures/DP-1.0-20161013.t.4/compose/Client/x86_64/iso/DP-1.0-20161013.t.4-Client-x86_64-dvd1.iso vendored Normal file
View File

1
tests/fixtures/DP-1.0-20161013.t.4/compose/Client/x86_64/iso/MD5SUM vendored Normal file
View File

@ -0,0 +1 @@
MD5 (DP-1.0-20161013.t.4-Client-x86_64-dvd1.iso) = 96f7e1e30fdc6681007f549ff01e4fe9

1
tests/fixtures/DP-1.0-20161013.t.4/compose/Client/x86_64/iso/SHA1SUM vendored Normal file
View File

@ -0,0 +1 @@
SHA1 (DP-1.0-20161013.t.4-Client-x86_64-dvd1.iso) = d4f158227510ad98d4f2f59e0885fccfb2813bee

1
tests/fixtures/DP-1.0-20161013.t.4/compose/Client/x86_64/iso/SHA256SUM vendored Normal file
View File

@ -0,0 +1 @@
SHA256 (DP-1.0-20161013.t.4-Client-x86_64-dvd1.iso) = 4baa0baf4f9e6c5e31c8fb278c0b4e7209ca6c041001d54c3c70f5110f54cbaa

33
tests/fixtures/DP-1.0-20161013.t.4/compose/Client/x86_64/os/.treeinfo vendored Normal file
View File

@ -0,0 +1,33 @@
[general]
; WARNING.0 = This section provides compatibility with pre-productmd treeinfos.
; WARNING.1 = Read productmd documentation for details about new format.
arch = x86_64
family = Dummy Product
name = Dummy Product 1.0
platforms = x86_64
timestamp = 1476343975
variant = Client
variants = Client
version = 1.0
[header]
type = productmd.treeinfo
version = 1.2
[release]
name = Dummy Product
short = DP
version = 1.0
[tree]
arch = x86_64
build_timestamp = 1476343975
platforms = x86_64
variants = Client
[variant-Client]
id = Client
name = Client
type = variant
uid = Client

0
tests/fixtures/DP-1.0-20161013.t.4/compose/Client/x86_64/os/Packages/d/dummy-bash-4.2.37-6.x86_64.rpm vendored Normal file
View File

0
tests/fixtures/DP-1.0-20161013.t.4/compose/Client/x86_64/os/repodata/comps-Client.x86_64.xml vendored Normal file
View File

0
tests/fixtures/DP-1.0-20161013.t.4/compose/Client/x86_64/os/repodata/repomd.xml vendored Normal file
View File

0
tests/fixtures/DP-1.0-20161013.t.4/compose/Server/s390x/iso/DP-1.0-20161013.t.4-Server-s390x-dvd1.iso vendored Normal file
View File

1
tests/fixtures/DP-1.0-20161013.t.4/compose/Server/s390x/iso/MD5SUM vendored Normal file
View File

@ -0,0 +1 @@
MD5 (DP-1.0-20161013.t.4-Server-s390x-dvd1.iso) = b236231385152239560a90d96746f44f

1
tests/fixtures/DP-1.0-20161013.t.4/compose/Server/s390x/iso/SHA1SUM vendored Normal file
View File

@ -0,0 +1 @@
SHA1 (DP-1.0-20161013.t.4-Server-s390x-dvd1.iso) = 6645d6f24c7c3e88709991597a4a80420a10e461

1
tests/fixtures/DP-1.0-20161013.t.4/compose/Server/s390x/iso/SHA256SUM vendored Normal file
View File

@ -0,0 +1 @@
SHA256 (DP-1.0-20161013.t.4-Server-s390x-dvd1.iso) = 2038d477509691e19940e545169328e9c9a9a2cc16a582d5fc82a89f4482b105

33
tests/fixtures/DP-1.0-20161013.t.4/compose/Server/s390x/os/.treeinfo vendored Normal file
View File

@ -0,0 +1,33 @@
[general]
; WARNING.0 = This section provides compatibility with pre-productmd treeinfos.
; WARNING.1 = Read productmd documentation for details about new format.
arch = s390x
family = Dummy Product
name = Dummy Product 1.0
platforms = s390x
timestamp = 1476343975
variant = Server
variants = Server
version = 1.0
[header]
type = productmd.treeinfo
version = 1.2
[release]
name = Dummy Product
short = DP
version = 1.0
[tree]
arch = s390x
build_timestamp = 1476343975
platforms = s390x
variants = Server
[variant-Server]
id = Server
name = Server
type = variant
uid = Server

0
tests/fixtures/DP-1.0-20161013.t.4/compose/Server/s390x/os/Packages/d/dummy-filesystem-4.2.37-6.s390x.rpm vendored Normal file
View File

0
tests/fixtures/DP-1.0-20161013.t.4/compose/Server/s390x/os/repodata/comps-Server.s390x.xml vendored Normal file
View File

0
tests/fixtures/DP-1.0-20161013.t.4/compose/Server/s390x/os/repodata/repomd.xml vendored Normal file
View File

0
tests/fixtures/DP-1.0-20161013.t.4/compose/Server/source/iso/DP-1.0-20161013.t.4-Server-source-dvd1.iso vendored Normal file
View File

1
tests/fixtures/DP-1.0-20161013.t.4/compose/Server/source/iso/MD5SUM vendored Normal file
View File

@ -0,0 +1 @@
MD5 (DP-1.0-20161013.t.4-Server-source-dvd1.iso) = 319e0e5e81da4726e8826d66405d4eff

1
tests/fixtures/DP-1.0-20161013.t.4/compose/Server/source/iso/SHA1SUM vendored Normal file
View File

@ -0,0 +1 @@
SHA1 (DP-1.0-20161013.t.4-Server-source-dvd1.iso) = f4620a41bce51db1ac99ca09c5ae74b91bea2869

1
tests/fixtures/DP-1.0-20161013.t.4/compose/Server/source/iso/SHA256SUM vendored Normal file
View File

@ -0,0 +1 @@
SHA256 (DP-1.0-20161013.t.4-Server-source-dvd1.iso) = 555c53081e0c8413b761898e436932d9066b3ca6794d7f042afae4388b3700f6

33
tests/fixtures/DP-1.0-20161013.t.4/compose/Server/source/tree/.treeinfo vendored Normal file
View File

@ -0,0 +1,33 @@
[general]
; WARNING.0 = This section provides compatibility with pre-productmd treeinfos.
; WARNING.1 = Read productmd documentation for details about new format.
arch = src
family = Dummy Product
name = Dummy Product 1.0
platforms = src
timestamp = 1476343975
variant = Server
variants = Server
version = 1.0
[header]
type = productmd.treeinfo
version = 1.2
[release]
name = Dummy Product
short = DP
version = 1.0
[tree]
arch = src
build_timestamp = 1476343975
platforms = src
variants = Server
[variant-Server]
id = Server
name = Server
type = variant
uid = Server

0
tests/fixtures/DP-1.0-20161013.t.4/compose/Server/source/tree/Packages/d/dummy-filesystem-4.2.37-6.src.rpm vendored Normal file
View File

0
tests/fixtures/DP-1.0-20161013.t.4/compose/Server/source/tree/repodata/repomd.xml vendored Normal file
View File

0
tests/fixtures/DP-1.0-20161013.t.4/compose/Server/x86_64/iso/DP-1.0-20161013.t.4-Server-x86_64-dvd1.iso vendored Normal file
View File

1
tests/fixtures/DP-1.0-20161013.t.4/compose/Server/x86_64/iso/MD5SUM vendored Normal file
View File

@ -0,0 +1 @@
MD5 (DP-1.0-20161013.t.4-Server-x86_64-dvd1.iso) = d2b474a3e7656781e23b4c1745e31c79

1
tests/fixtures/DP-1.0-20161013.t.4/compose/Server/x86_64/iso/SHA1SUM vendored Normal file
View File

@ -0,0 +1 @@
SHA1 (DP-1.0-20161013.t.4-Server-x86_64-dvd1.iso) = c5d91e52dec6b7edd9d5eeaf6e1926c97b5d7a3d

1
tests/fixtures/DP-1.0-20161013.t.4/compose/Server/x86_64/iso/SHA256SUM vendored Normal file
View File

@ -0,0 +1 @@
SHA256 (DP-1.0-20161013.t.4-Server-x86_64-dvd1.iso) = a435be0622d9e61febdad160907967303fbf381350cd5ee4d31916f403f4fb28

33
tests/fixtures/DP-1.0-20161013.t.4/compose/Server/x86_64/os/.treeinfo vendored Normal file
View File

@ -0,0 +1,33 @@
[general]
; WARNING.0 = This section provides compatibility with pre-productmd treeinfos.
; WARNING.1 = Read productmd documentation for details about new format.
arch = x86_64
family = Dummy Product
name = Dummy Product 1.0
platforms = x86_64
timestamp = 1476343975
variant = Server
variants = Server
version = 1.0
[header]
type = productmd.treeinfo
version = 1.2
[release]
name = Dummy Product
short = DP
version = 1.0
[tree]
arch = x86_64
build_timestamp = 1476343975
platforms = x86_64
variants = Server
[variant-Server]
id = Server
name = Server
type = variant
uid = Server

0
tests/fixtures/DP-1.0-20161013.t.4/compose/Server/x86_64/os/Packages/d/dummy-filesystem-4.2.37-6.x86_64.rpm vendored Normal file
View File

0
tests/fixtures/DP-1.0-20161013.t.4/compose/Server/x86_64/os/repodata/comps-Server.x86_64.xml vendored Normal file
View File

0
tests/fixtures/DP-1.0-20161013.t.4/compose/Server/x86_64/os/repodata/repomd.xml vendored Normal file
View File

135
tests/fixtures/DP-1.0-20161013.t.4/compose/metadata/composeinfo.json vendored Normal file
View File

@ -0,0 +1,135 @@
{
"header": {
"type": "productmd.composeinfo",
"version": "1.2"
},
"payload": {
"compose": {
"date": "20161013",
"id": "DP-1.0-20161013.t.4",
"respin": 4,
"type": "test"
},
"release": {
"internal": false,
"name": "Dummy Product",
"short": "DP",
"type": "ga",
"version": "1.0"
},
"variants": {
"Client": {
"arches": [
"i386",
"x86_64"
],
"id": "Client",
"name": "Client",
"paths": {
"debug_packages": {
"i386": "Client/i386/debug/tree/Packages",
"x86_64": "Client/x86_64/debug/tree/Packages"
},
"debug_repository": {
"i386": "Client/i386/debug/tree",
"x86_64": "Client/x86_64/debug/tree"
},
"debug_tree": {
"i386": "Client/i386/debug/tree",
"x86_64": "Client/x86_64/debug/tree"
},
"isos": {
"i386": "Client/i386/iso",
"x86_64": "Client/x86_64/iso"
},
"os_tree": {
"i386": "Client/i386/os",
"x86_64": "Client/x86_64/os"
},
"packages": {
"i386": "Client/i386/os/Packages",
"x86_64": "Client/x86_64/os/Packages"
},
"repository": {
"i386": "Client/i386/os",
"x86_64": "Client/x86_64/os"
},
"source_isos": {
"i386": "Client/source/iso",
"x86_64": "Client/source/iso"
},
"source_packages": {
"i386": "Client/source/tree/Packages",
"x86_64": "Client/source/tree/Packages"
},
"source_repository": {
"i386": "Client/source/tree",
"x86_64": "Client/source/tree"
},
"source_tree": {
"i386": "Client/source/tree",
"x86_64": "Client/source/tree"
}
},
"type": "variant",
"uid": "Client"
},
"Server": {
"arches": [
"s390x",
"x86_64"
],
"id": "Server",
"name": "Server",
"paths": {
"debug_packages": {
"s390x": "Server/s390x/debug/tree/Packages",
"x86_64": "Server/x86_64/debug/tree/Packages"
},
"debug_repository": {
"s390x": "Server/s390x/debug/tree",
"x86_64": "Server/x86_64/debug/tree"
},
"debug_tree": {
"s390x": "Server/s390x/debug/tree",
"x86_64": "Server/x86_64/debug/tree"
},
"isos": {
"s390x": "Server/s390x/iso",
"x86_64": "Server/x86_64/iso"
},
"os_tree": {
"s390x": "Server/s390x/os",
"x86_64": "Server/x86_64/os"
},
"packages": {
"s390x": "Server/s390x/os/Packages",
"x86_64": "Server/x86_64/os/Packages"
},
"repository": {
"s390x": "Server/s390x/os",
"x86_64": "Server/x86_64/os"
},
"source_isos": {
"s390x": "Server/source/iso",
"x86_64": "Server/source/iso"
},
"source_packages": {
"s390x": "Server/source/tree/Packages",
"x86_64": "Server/source/tree/Packages"
},
"source_repository": {
"s390x": "Server/source/tree",
"x86_64": "Server/source/tree"
},
"source_tree": {
"s390x": "Server/source/tree",
"x86_64": "Server/source/tree"
}
},
"type": "variant",
"uid": "Server"
}
}
}
}

180
tests/fixtures/DP-1.0-20161013.t.4/compose/metadata/images.json vendored Normal file
View File

@ -0,0 +1,180 @@
{
"header": {
"type": "productmd.images",
"version": "1.2"
},
"payload": {
"compose": {
"date": "20161013",
"id": "DP-1.0-20161013.t.4",
"respin": 4,
"type": "test"
},
"images": {
"Client": {
"i386": [
{
"arch": "i386",
"bootable": false,
"checksums": {
"md5": "d41d8cd98f00b204e9800998ecf8427e",
"sha1": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
"disc_count": 1,
"disc_number": 1,
"format": "iso",
"implant_md5": "cd55df8932aa40c992392a12d867622d",
"mtime": 1476343975,
"path": "Client/i386/iso/DP-1.0-20161013.t.4-Client-i386-dvd1.iso",
"size": 505856,
"subvariant": "Client",
"type": "dvd",
"volume_id": "DP-1.0 Client.i386"
},
{
"arch": "src",
"bootable": false,
"checksums": {
"md5": "d41d8cd98f00b204e9800998ecf8427e",
"sha1": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
"disc_count": 1,
"disc_number": 1,
"format": "iso",
"implant_md5": "8f14699db503e223ecce265f050bdcb2",
"mtime": 1476343975,
"path": "Client/source/iso/DP-1.0-20161013.t.4-Client-source-dvd1.iso",
"size": 473088,
"subvariant": "Client",
"type": "dvd",
"volume_id": "DP-1.0 Client.src"
}
],
"x86_64": [
{
"arch": "src",
"bootable": false,
"checksums": {
"md5": "d41d8cd98f00b204e9800998ecf8427e",
"sha1": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
"disc_count": 1,
"disc_number": 1,
"format": "iso",
"implant_md5": "8f14699db503e223ecce265f050bdcb2",
"mtime": 1476343975,
"path": "Client/source/iso/DP-1.0-20161013.t.4-Client-source-dvd1.iso",
"size": 473088,
"subvariant": "Client",
"type": "dvd",
"volume_id": "DP-1.0 Client.src"
},
{
"arch": "x86_64",
"bootable": false,
"checksums": {
"md5": "d41d8cd98f00b204e9800998ecf8427e",
"sha1": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
"disc_count": 1,
"disc_number": 1,
"format": "iso",
"implant_md5": "ae708b67ba4594fd57fc390057a73ee2",
"mtime": 1476343975,
"path": "Client/x86_64/iso/DP-1.0-20161013.t.4-Client-x86_64-dvd1.iso",
"size": 516096,
"subvariant": "Client",
"type": "dvd",
"volume_id": "DP-1.0 Client.x86_64"
}
]
},
"Server": {
"s390x": [
{
"arch": "s390x",
"bootable": false,
"checksums": {
"md5": "d41d8cd98f00b204e9800998ecf8427e",
"sha1": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
"disc_count": 1,
"disc_number": 1,
"format": "iso",
"implant_md5": "7ca8b25c099330d3cc7e9fa0e610745a",
"mtime": 1476343975,
"path": "Server/s390x/iso/DP-1.0-20161013.t.4-Server-s390x-dvd1.iso",
"size": 493568,
"subvariant": "Server",
"type": "dvd",
"volume_id": "DP-1.0 Server.s390x"
},
{
"arch": "src",
"bootable": false,
"checksums": {
"md5": "d41d8cd98f00b204e9800998ecf8427e",
"sha1": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
"disc_count": 1,
"disc_number": 1,
"format": "iso",
"implant_md5": "b03d9e01da352cbe01ba4906bc9ce3ca",
"mtime": 1476343975,
"path": "Server/source/iso/DP-1.0-20161013.t.4-Server-source-dvd1.iso",
"size": 458752,
"subvariant": "Server",
"type": "dvd",
"volume_id": "DP-1.0 Server.src"
}
],
"x86_64": [
{
"arch": "src",
"bootable": false,
"checksums": {
"md5": "d41d8cd98f00b204e9800998ecf8427e",
"sha1": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
"disc_count": 1,
"disc_number": 1,
"format": "iso",
"implant_md5": "b03d9e01da352cbe01ba4906bc9ce3ca",
"mtime": 1476343975,
"path": "Server/source/iso/DP-1.0-20161013.t.4-Server-source-dvd1.iso",
"size": 458752,
"subvariant": "Server",
"type": "dvd",
"volume_id": "DP-1.0 Server.src"
},
{
"arch": "x86_64",
"bootable": false,
"checksums": {
"md5": "d41d8cd98f00b204e9800998ecf8427e",
"sha1": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
"disc_count": 1,
"disc_number": 1,
"format": "iso",
"implant_md5": "4e4c4c67e158b507e1084b7bd925cee1",
"mtime": 1476343975,
"path": "Server/x86_64/iso/DP-1.0-20161013.t.4-Server-x86_64-dvd1.iso",
"size": 493568,
"subvariant": "Server",
"type": "dvd",
"volume_id": "DP-1.0 Server.x86_64"
}
]
}
}
}
}

164
tests/fixtures/DP-1.0-20161013.t.4/logs/global/config-dump_2016-10-13_09-32-46.global.log vendored Normal file
View File

@ -0,0 +1,164 @@
{
"additional_packages": [
[
"^Server$",
{
"*": [
"dummy-libtool"
]
}
],
[
"^Client-optional$",
{
"*": [
"dummy-httpd"
]
}
]
],
"base_product_type": "ga",
"bootable": false,
"check_deps": false,
"comps_file": "dummy-comps.xml",
"comps_filter_environments": true,
"create_jigdo": false,
"create_optional_isos": false,
"createiso_skip": [
[
"^Server-Gluster$",
{
"*": true,
"src": true
}
]
],
"createrepo_c": true,
"createrepo_checksum": "sha256",
"createrepo_deltas": false,
"createrepo_use_xz": false,
"disc_types": {},
"extra_files": [],
"failable_deliverables": [],
"filter_packages": [
[
"^.*$",
{
"*": [
"dummy-pacemaker"
]
}
],
[
"^Client$",
{
"*": [
"dummy-httpd"
]
}
],
[
"^Server-optional$",
{
"*": [
"dummy-httpd.i686"
]
}
],
[
"^.*-ResilientStorage$",
{
"*": [
"dummy-glusterfs-resource-agents"
]
}
]
],
"filter_system_release_packages": true,
"gather_fulltree": false,
"gather_lookaside_repos": [],
"gather_method": "deps",
"gather_selfhosting": false,
"gather_source": "comps",
"greedy_method": "build",
"hashed_directories": true,
"image_volid_formats": [
"{release_short}-{version} {variant}.{arch}",
"{release_short}-{version} {arch}"
],
"image_volid_layered_product_formats": [
"{release_short}-{version} {base_product_short}-{base_product_version} {variant}.{arch}",
"{release_short}-{version} {base_product_short}-{base_product_version} {arch}"
],
"iso_size": 4700000000,
"link_type": "hardlink-or-copy",
"live_images": [],
"live_images_no_rename": false,
"live_target": "rhel-7.0-candidate",
"lorax_options": [],
"media_checksum_base_filename": "",
"media_checksum_one_file": false,
"media_checksums": [
"md5",
"sha1",
"sha256"
],
"multilib": [
[
"^Server.*$",
{
"*": [
"devel",
"runtime"
]
}
]
],
"multilib_blacklist": {
"*": [
"kernel-devel",
"httpd-devel"
]
},
"multilib_whitelist": {
"*": [
"dummy-glibc"
]
},
"ostree": [],
"ostree_installer": [],
"pkgset_koji_inherit": true,
"pkgset_repos": {
"i386": [
"repo"
],
"s390x": [
"repo"
],
"x86_64": [
"repo"
]
},
"pkgset_source": "repos",
"product_id_allow_missing": false,
"productimg": false,
"release_is_layered": false,
"release_name": "Dummy Product",
"release_short": "DP",
"release_type": "ga",
"release_version": "1.0",
"runroot": false,
"sigkeys": [
null
],
"skip_phases": [],
"split_iso_reserve": 10485760,
"translate_paths": [],
"tree_arches": [],
"tree_variants": [
"Client",
"Server"
],
"variants_file": "dummy-variants.xml",
"volume_id_substitutions": {}
}

0
tests/fixtures/DP-1.0-20161013.t.4/work/.gitignore vendored Normal file
View File

1
tests/helpers.py
View File

@ -117,6 +117,7 @@ def touch(path, content=None):
pass
with open(path, 'w') as f:
f.write(content)
return path
FIXTURE_DIR = os.path.join(os.path.dirname(__file__), 'fixtures')

10
tests/test_compose.sh
View File

@ -1,7 +1,10 @@
#!/bin/sh
export PYTHONPATH=$(pwd)/../:$PYTHONPATH
export PATH=$(pwd)/../bin:$PATH
set -e
PYTHONPATH=$(pwd)/../:$PYTHONPATH
PATH=$(pwd)/../bin:$PATH
export PYTHONPATH PATH
mkdir -p _composes
@ -10,3 +13,6 @@ pungi-koji \
--old-composes=_composes \
--config=data/dummy-pungi.conf \
--test "$@"
# Run this to create unified ISOs for the just created compose
#pungi-create-unified-isos _composes/latest-DP-1/

544
tests/test_unified_isos.py Executable file
View File

@ -0,0 +1,544 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import json
import mock
import os
import shutil
import sys
from ConfigParser import SafeConfigParser
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
from tests.helpers import PungiTestCase, FIXTURE_DIR, touch
from pungi_utils import unified_isos
COMPOSE_ID = 'DP-1.0-20161013.t.4'
class TestUnifiedIsos(PungiTestCase):
def setUp(self):
super(TestUnifiedIsos, self).setUp()
shutil.copytree(os.path.join(FIXTURE_DIR, COMPOSE_ID),
os.path.join(self.topdir, COMPOSE_ID))
def test_can_init(self):
compose_path = os.path.join(self.topdir, COMPOSE_ID, 'compose')
isos = unified_isos.UnifiedISO(compose_path)
self.assertEqual(isos.compose_path, compose_path)
self.assertRegexpMatches(isos.temp_dir,
'^%s/' % os.path.join(self.topdir, COMPOSE_ID, 'work'))
def test_can_find_compose_subdir(self):
isos = unified_isos.UnifiedISO(os.path.join(self.topdir, COMPOSE_ID))
self.assertEqual(isos.compose_path, os.path.join(self.topdir, COMPOSE_ID, 'compose'))
self.assertRegexpMatches(isos.temp_dir,
'^%s/' % os.path.join(self.topdir, COMPOSE_ID, 'work'))
class TestCreate(PungiTestCase):
def setUp(self):
super(TestCreate, self).setUp()
shutil.copytree(os.path.join(FIXTURE_DIR, COMPOSE_ID),
os.path.join(self.topdir, COMPOSE_ID))
compose_path = os.path.join(self.topdir, COMPOSE_ID, 'compose')
self.isos = unified_isos.UnifiedISO(compose_path)
def test_create_method(self):
methods = ('link_to_temp', 'createrepo', 'discinfo', 'createiso',
'link_to_compose', 'update_checksums')
for attr in methods:
setattr(self.isos, attr, mock.Mock())
with mock.patch('shutil.rmtree') as rmtree:
self.isos.create()
for attr in methods:
self.assertEqual(len(getattr(self.isos, attr).call_args_list), 1)
self.assertEqual(rmtree.call_args_list,
[mock.call(self.isos.temp_dir)])
def get_comps_mapping(path):
def _comps(variant, arch):
return os.path.join(path, variant, arch, 'os', 'repodata',
'comps-%s.%s.xml' % (variant, arch))
return {
'i386': {'Client': _comps('Client', 'i386')},
's390x': {'Server': _comps('Server', 's390x')},
'x86_64': {'Client': _comps('Client', 'x86_64'),
'Server': _comps('Server', 'x86_64')}
}
def get_productid_mapping(path):
def _productid(variant, arch):
return os.path.join(path, variant, arch, 'os', 'repodata', 'productid')
return {
'i386': {'Client': _productid('Client', 'i386')},
's390x': {'Server': _productid('Server', 's390x')},
'x86_64': {'Client': _productid('Client', 'x86_64'),
'Server': _productid('Server', 'x86_64')}
}
def get_repos_mapping(path):
def _repo(variant, arch):
return os.path.join(path, 'trees', arch, variant)
return {
'i386': {'Client': _repo('Client', 'i386')},
's390x': {'Server': _repo('Server', 's390x')},
'src': {'Client': _repo('Client', 'src'),
'Server': _repo('Server', 'src')},
'x86_64': {'Client': _repo('Client', 'x86_64'),
'Server': _repo('Server', 'x86_64')}
}
class TestLinkToTemp(PungiTestCase):
def setUp(self):
super(TestLinkToTemp, self).setUp()
shutil.copytree(os.path.join(FIXTURE_DIR, COMPOSE_ID),
os.path.join(self.topdir, COMPOSE_ID))
self.compose_path = os.path.join(self.topdir, COMPOSE_ID, 'compose')
self.isos = unified_isos.UnifiedISO(self.compose_path)
self.isos.linker = mock.Mock()
def _linkCall(self, variant, arch, file):
return mock.call(os.path.join(self.compose_path, variant,
arch if arch != 'src' else 'source',
'tree' if arch == 'src' else 'os',
'Packages', file[0].lower(), file),
os.path.join(self.isos.temp_dir, 'trees', arch, variant, file))
def test_link_to_temp(self):
self.isos.link_to_temp()
self.assertItemsEqual(self.isos.treeinfo.keys(),
['i386', 's390x', 'src', 'x86_64'])
self.assertEqual(self.isos.comps,
get_comps_mapping(self.compose_path))
self.assertEqual(self.isos.productid,
get_productid_mapping(self.compose_path))
self.assertEqual(self.isos.repos,
get_repos_mapping(self.isos.temp_dir))
self.assertItemsEqual(self.isos.linker.link.call_args_list,
[self._linkCall('Server', 's390x', 'dummy-filesystem-4.2.37-6.s390x.rpm'),
self._linkCall('Server', 'x86_64', 'dummy-filesystem-4.2.37-6.x86_64.rpm'),
self._linkCall('Server', 'src', 'dummy-filesystem-4.2.37-6.src.rpm'),
self._linkCall('Server', 'src', 'dummy-filesystem-4.2.37-6.src.rpm'),
self._linkCall('Client', 'i386', 'dummy-bash-4.2.37-6.i686.rpm'),
self._linkCall('Client', 'x86_64', 'dummy-bash-4.2.37-6.x86_64.rpm'),
self._linkCall('Client', 'src', 'dummy-bash-4.2.37-6.src.rpm'),
self._linkCall('Client', 'src', 'dummy-bash-4.2.37-6.src.rpm')])
def test_link_to_temp_without_treefile(self):
os.remove(os.path.join(self.compose_path, 'Client', 'i386', 'os', '.treeinfo'))
with mock.patch('sys.stderr'):
self.isos.link_to_temp()
self.assertItemsEqual(self.isos.treeinfo.keys(),
['s390x', 'src', 'x86_64'])
comps = get_comps_mapping(self.compose_path)
comps.pop('i386')
self.assertEqual(self.isos.comps, comps)
productid = get_productid_mapping(self.compose_path)
productid.pop('i386')
self.assertEqual(self.isos.productid, productid)
repos = get_repos_mapping(self.isos.temp_dir)
repos.pop('i386')
self.assertEqual(self.isos.repos, repos)
self.assertItemsEqual(self.isos.linker.link.call_args_list,
[self._linkCall('Server', 's390x', 'dummy-filesystem-4.2.37-6.s390x.rpm'),
self._linkCall('Server', 'x86_64', 'dummy-filesystem-4.2.37-6.x86_64.rpm'),
self._linkCall('Server', 'src', 'dummy-filesystem-4.2.37-6.src.rpm'),
self._linkCall('Server', 'src', 'dummy-filesystem-4.2.37-6.src.rpm'),
self._linkCall('Client', 'x86_64', 'dummy-bash-4.2.37-6.x86_64.rpm'),
self._linkCall('Client', 'src', 'dummy-bash-4.2.37-6.src.rpm')])
def test_link_to_temp_extra_file(self):
gpl_file = touch(os.path.join(self.compose_path, 'Server', 'x86_64', 'os', 'GPL'))
self.isos.link_to_temp()
self.assertItemsEqual(self.isos.treeinfo.keys(),
['i386', 's390x', 'src', 'x86_64'])
self.assertEqual(self.isos.comps,
get_comps_mapping(self.compose_path))
self.assertEqual(self.isos.productid,
get_productid_mapping(self.compose_path))
self.assertEqual(self.isos.repos,
get_repos_mapping(self.isos.temp_dir))
self.assertItemsEqual(self.isos.linker.link.call_args_list,
[self._linkCall('Server', 's390x', 'dummy-filesystem-4.2.37-6.s390x.rpm'),
self._linkCall('Server', 'x86_64', 'dummy-filesystem-4.2.37-6.x86_64.rpm'),
self._linkCall('Server', 'src', 'dummy-filesystem-4.2.37-6.src.rpm'),
self._linkCall('Server', 'src', 'dummy-filesystem-4.2.37-6.src.rpm'),
self._linkCall('Client', 'i386', 'dummy-bash-4.2.37-6.i686.rpm'),
self._linkCall('Client', 'x86_64', 'dummy-bash-4.2.37-6.x86_64.rpm'),
self._linkCall('Client', 'src', 'dummy-bash-4.2.37-6.src.rpm'),
self._linkCall('Client', 'src', 'dummy-bash-4.2.37-6.src.rpm'),
mock.call(os.path.join(gpl_file),
os.path.join(self.isos.temp_dir, 'trees', 'x86_64', 'GPL'))])
class TestCreaterepo(PungiTestCase):
def setUp(self):
super(TestCreaterepo, self).setUp()
shutil.copytree(os.path.join(FIXTURE_DIR, COMPOSE_ID),
os.path.join(self.topdir, COMPOSE_ID))
self.compose_path = os.path.join(self.topdir, COMPOSE_ID, 'compose')
self.isos = unified_isos.UnifiedISO(self.compose_path)
self.isos.linker = mock.Mock()
# TODO mock treeinfo and use mappings for other data
self.isos.link_to_temp()
self.maxDiff = None
self.comps = get_comps_mapping(self.compose_path)
def mock_cr(self, path, groupfile, update):
self.assertTrue(update)
touch(os.path.join(path, 'repodata', 'repomd.xml'))
return ('/'.join(path.split('/')[-2:]), groupfile)
def mock_mr(self, path, pid, compress_type):
self.assertEqual(compress_type, 'gz')
return ('/'.join(path.split('/')[-3:-1]), pid)
@mock.patch('pungi.wrappers.createrepo.CreaterepoWrapper')
@mock.patch('pungi_utils.unified_isos.run')
def test_createrepo(self, run, cr):
cr.return_value.get_createrepo_cmd.side_effect = self.mock_cr
self.isos.createrepo()
self.assertItemsEqual(
run.call_args_list,
[mock.call(('src/Client', None), show_cmd=True),
mock.call(('src/Server', None), show_cmd=True),
mock.call(('i386/Client', self.comps['i386']['Client']), show_cmd=True),
mock.call(('s390x/Server', self.comps['s390x']['Server']), show_cmd=True),
mock.call(('x86_64/Client', self.comps['x86_64']['Client']), show_cmd=True),
mock.call(('x86_64/Server', self.comps['x86_64']['Server']), show_cmd=True)]
)
checksums = {}
# treeinfo checksums
for arch in self.isos.treeinfo.keys():
parser = SafeConfigParser()
parser.optionxform = str
parser.read(os.path.join(self.isos.temp_dir, 'trees', arch, '.treeinfo'))
checksums[arch] = [k for k, v in parser.items('checksums')]
self.assertEqual(
checksums,
{'i386': ['Client/repodata/repomd.xml'],
's390x': ['Server/repodata/repomd.xml'],
'src': ['Client/repodata/repomd.xml', 'Server/repodata/repomd.xml'],
'x86_64': ['Client/repodata/repomd.xml', 'Server/repodata/repomd.xml']}
)
@mock.patch('pungi.wrappers.createrepo.CreaterepoWrapper')
@mock.patch('pungi_utils.unified_isos.run')
def test_createrepo_with_productid(self, run, cr):
for x in self.isos.productid.values():
for f in x.values():
touch(f)
cr.return_value.get_createrepo_cmd.side_effect = self.mock_cr
cr.return_value.get_modifyrepo_cmd.side_effect = self.mock_mr
self.isos.createrepo()
self.assertItemsEqual(
run.call_args_list,
[mock.call(('src/Client', None), show_cmd=True),
mock.call(('src/Server', None), show_cmd=True),
mock.call(('i386/Client', self.comps['i386']['Client']), show_cmd=True),
mock.call(('s390x/Server', self.comps['s390x']['Server']), show_cmd=True),
mock.call(('x86_64/Client', self.comps['x86_64']['Client']), show_cmd=True),
mock.call(('x86_64/Server', self.comps['x86_64']['Server']), show_cmd=True),
mock.call(('x86_64/Server', os.path.join(self.isos.temp_dir,
'trees/x86_64/Server/repodata/productid'))),
mock.call(('x86_64/Client', os.path.join(self.isos.temp_dir,
'trees/x86_64/Client/repodata/productid'))),
mock.call(('s390x/Server', os.path.join(self.isos.temp_dir,
'trees/s390x/Server/repodata/productid'))),
mock.call(('i386/Client', os.path.join(self.isos.temp_dir,
'trees/i386/Client/repodata/productid')))]
)
checksums = {}
# treeinfo checksums
for arch in self.isos.treeinfo.keys():
parser = SafeConfigParser()
parser.optionxform = str
parser.read(os.path.join(self.isos.temp_dir, 'trees', arch, '.treeinfo'))
checksums[arch] = [k for k, v in parser.items('checksums')]
self.assertEqual(
checksums,
{'i386': ['Client/repodata/repomd.xml'],
's390x': ['Server/repodata/repomd.xml'],
'src': ['Client/repodata/repomd.xml', 'Server/repodata/repomd.xml'],
'x86_64': ['Client/repodata/repomd.xml', 'Server/repodata/repomd.xml']}
)
class TestDiscinfo(PungiTestCase):
def setUp(self):
super(TestDiscinfo, self).setUp()
shutil.copytree(os.path.join(FIXTURE_DIR, COMPOSE_ID),
os.path.join(self.topdir, COMPOSE_ID))
self.compose_path = os.path.join(self.topdir, COMPOSE_ID, 'compose')
self.isos = unified_isos.UnifiedISO(self.compose_path)
self.isos.linker = mock.Mock()
# TODO mock treeinfo and use mappings for other data
self.isos.link_to_temp()
self.maxDiff = None
@mock.patch('pungi_utils.unified_isos.create_discinfo')
def test_discinfo(self, create_discinfo):
self.isos.discinfo()
self.assertItemsEqual(
create_discinfo.call_args_list,
[mock.call(os.path.join(self.isos.temp_dir, 'trees', 'i386', '.discinfo'),
'Dummy Product 1.0', 'i386'),
mock.call(os.path.join(self.isos.temp_dir, 'trees', 's390x', '.discinfo'),
'Dummy Product 1.0', 's390x'),
mock.call(os.path.join(self.isos.temp_dir, 'trees', 'src', '.discinfo'),
'Dummy Product 1.0', 'src'),
mock.call(os.path.join(self.isos.temp_dir, 'trees', 'x86_64', '.discinfo'),
'Dummy Product 1.0', 'x86_64')]
)
CHECKSUMS = {
'MD5': 'cbc3a5767b22babfe3578a2b82d83fcb',
'SHA1': 'afaf8621bfbc22781edfc81b774a2b2f66fdc8b0',
'SHA256': '84c1c8611b287209e1e76d657e7e69e6192ad72dd2531e0fb7a43b95070fabb1',
}
class TestCreateiso(PungiTestCase):
def setUp(self):
super(TestCreateiso, self).setUp()
shutil.copytree(os.path.join(FIXTURE_DIR, COMPOSE_ID),
os.path.join(self.topdir, COMPOSE_ID))
self.compose_path = os.path.join(self.topdir, COMPOSE_ID, 'compose')
self.isos = unified_isos.UnifiedISO(self.compose_path)
self.isos.linker = mock.Mock()
# TODO mock treeinfo and use mappings for other data
self.isos.link_to_temp()
self.maxDiff = None
self.mkisofs_cmd = None
def mock_gmc(self, path, *args, **kwargs):
touch(path, 'ISO FILE\n')
self.mkisofs_cmd = self.mkisofs_cmd or mock.Mock(name='mkisofs cmd')
return self.mkisofs_cmd
def _img(self, arch, exts):
exts = ['manifest'] + exts
base_path = os.path.join(self.isos.temp_dir, 'iso', arch,
u'DP-1.0-20161013.t.4-%s-dvd.iso' % arch)
yield base_path
for ext in exts:
yield base_path + '.' + ext
def _imgs(self, arches, exts):
images = {}
exts = [e + 'SUM' for e in exts]
for arch in arches:
images[arch] = set(self._img(arch if arch != 'src' else 'source', exts))
return images
def assertResults(self, iso, run, arches, checksums):
self.assertEqual(
run.mock_calls,
[mock.call(self.mkisofs_cmd),
mock.call(iso.get_implantisomd5_cmd.return_value),
mock.call(iso.get_manifest_cmd.return_value)] * 2
)
self.assertEqual(
self.isos.images,
self._imgs(arches, checksums),
)
with open(os.path.join(self.compose_path, 'metadata', 'images.json')) as f:
manifest = json.load(f)
for v in ('Client', 'Server'):
for a in arches:
for image in manifest['payload']['images'][v]['x86_64']:
if image.get('unified', False) and image['arch'] == a:
arch = 'source' if image['arch'] == 'src' else image['arch']
self.assertEqual(image['path'],
'{0}/{1}/iso/DP-1.0-20161013.t.4-{1}-dvd.iso'.format(v, arch))
checksum_file_base = os.path.join(self.isos.temp_dir, 'iso',
arch, os.path.basename(image['path']))
for ch in checksums:
fp = '%s.%sSUM' % (checksum_file_base, ch)
with open(fp) as f:
self.assertEqual(
f.read(),
'%s (%s) = %s\n' % (ch, os.path.basename(image['path']),
CHECKSUMS[ch])
)
break
else:
self.fail('Image for %s.%s missing' % (v, a))
@mock.patch('pungi_utils.unified_isos.iso')
@mock.patch('pungi_utils.unified_isos.run')
def test_createiso(self, run, iso):
iso.get_mkisofs_cmd.side_effect = self.mock_gmc
iso.get_implanted_md5.return_value = 'beefcafebabedeadbeefcafebabedead'
iso.get_volume_id.return_value = 'VOLID'
self.isos.treeinfo = {'x86_64': self.isos.treeinfo['x86_64'],
'src': self.isos.treeinfo['src']}
self.isos.createiso()
self.assertResults(iso, run, ['src', 'x86_64'], ['MD5', 'SHA1', 'SHA256'])
@mock.patch('pungi_utils.unified_isos.iso')
@mock.patch('pungi_utils.unified_isos.run')
def test_createiso_checksum_one_file(self, run, iso):
iso.get_mkisofs_cmd.side_effect = self.mock_gmc
iso.get_implanted_md5.return_value = 'beefcafebabedeadbeefcafebabedead'
iso.get_volume_id.return_value = 'VOLID'
self.isos.conf['media_checksum_one_file'] = True
self.isos.treeinfo = {'x86_64': self.isos.treeinfo['x86_64'],
'src': self.isos.treeinfo['src']}
self.isos.createiso()
self.assertResults(iso, run, ['src', 'x86_64'], [])
@mock.patch('pungi_utils.unified_isos.iso')
@mock.patch('pungi_utils.unified_isos.run')
def test_createiso_single_checksum(self, run, iso):
iso.get_mkisofs_cmd.side_effect = self.mock_gmc
iso.get_implanted_md5.return_value = 'beefcafebabedeadbeefcafebabedead'
iso.get_volume_id.return_value = 'VOLID'
self.isos.conf['media_checksums'] = ['sha256']
self.isos.treeinfo = {'x86_64': self.isos.treeinfo['x86_64'],
'src': self.isos.treeinfo['src']}
self.isos.createiso()
self.assertResults(iso, run, ['src', 'x86_64'], ['SHA256'])
class TestLinkToCompose(PungiTestCase):
def setUp(self):
super(TestLinkToCompose, self).setUp()
shutil.copytree(os.path.join(FIXTURE_DIR, COMPOSE_ID),
os.path.join(self.topdir, COMPOSE_ID))
self.compose_path = os.path.join(self.topdir, COMPOSE_ID, 'compose')
self.isos = unified_isos.UnifiedISO(self.compose_path)
self.isos.linker = mock.Mock()
self.binary = os.path.join(self.isos.temp_dir, 'isos', 'x86_64', 'binary.iso')
self.source = os.path.join(self.isos.temp_dir, 'isos', 'src', 'source.iso')
self.isos.images = {
'x86_64': set([self.binary]),
'src': set([self.source]),
}
self.maxDiff = None
def _iso(self, variant, arch, name):
return os.path.join(self.compose_path, variant, arch, 'iso', name)
def test_link_to_compose(self):
self.isos.link_to_compose()
self.assertItemsEqual(
self.isos.linker.link.call_args_list,
[mock.call(self.binary, self._iso('Client', 'x86_64', 'binary.iso')),
mock.call(self.binary, self._iso('Server', 'x86_64', 'binary.iso')),
mock.call(self.source, self._iso('Client', 'source', 'source.iso')),
mock.call(self.source, self._iso('Server', 'source', 'source.iso'))]
)
class MockImage(mock.Mock):
def __eq__(self, another):
return self.path == another.path
class TestUpdateChecksums(PungiTestCase):
def setUp(self):
super(TestUpdateChecksums, self).setUp()
shutil.copytree(os.path.join(FIXTURE_DIR, COMPOSE_ID),
os.path.join(self.topdir, COMPOSE_ID))
self.compose_path = os.path.join(self.topdir, COMPOSE_ID, 'compose')
self.isos = unified_isos.UnifiedISO(self.compose_path)
self.maxDiff = None
def _isodir(self, variant, arch):
return os.path.join(self.compose_path, variant, arch, 'iso')
def _call(self, variant, arch, source=False, basename='', one_file=False):
archdir = arch if not source else 'source'
isodir = self._isodir(variant, archdir)
filename = 'DP-1.0-20161013.t.4-%s-%s-dvd1.iso' % (variant, archdir)
return mock.call(variant, arch, isodir,
[MockImage(path=os.path.join(variant, archdir, 'iso', filename))],
['md5', 'sha1', 'sha256'], basename, one_file)
@mock.patch('pungi_utils.unified_isos.make_checksums')
def test_update_checksums(self, mmc):
self.isos.update_checksums()
self.assertItemsEqual(
mmc.call_args_list,
[self._call('Client', 'i386'),
self._call('Client', 'x86_64'),
self._call('Server', 's390x'),
self._call('Server', 'x86_64'),
self._call('Client', 'i386', source=True),
self._call('Client', 'x86_64', source=True),
self._call('Server', 's390x', source=True),
self._call('Server', 'x86_64', source=True)]
)
@mock.patch('pungi_utils.unified_isos.make_checksums')
def test_update_checksums_one_file(self, mmc):
self.isos.conf['media_checksum_one_file'] = True
self.isos.update_checksums()
self.assertItemsEqual(
mmc.call_args_list,
[self._call('Client', 'i386', one_file=True),
self._call('Client', 'x86_64', one_file=True),
self._call('Server', 's390x', one_file=True),
self._call('Server', 'x86_64', one_file=True),
self._call('Client', 'i386', source=True, one_file=True),
self._call('Client', 'x86_64', source=True, one_file=True),
self._call('Server', 's390x', source=True, one_file=True),
self._call('Server', 'x86_64', source=True, one_file=True)]
)
@mock.patch('pungi_utils.unified_isos.make_checksums')
def test_update_checksums_basename(self, mmc):
self.isos.conf['media_checksum_base_filename'] = '{variant}-{arch}'
self.isos.update_checksums()
self.assertItemsEqual(
mmc.call_args_list,
[self._call('Client', 'i386', basename='Client-i386-'),
self._call('Client', 'x86_64', basename='Client-x86_64-'),
self._call('Server', 's390x', basename='Server-s390x-'),
self._call('Server', 'x86_64', basename='Server-x86_64-'),
self._call('Client', 'i386', source=True, basename='Client-i386-'),
self._call('Client', 'x86_64', source=True, basename='Client-x86_64-'),
self._call('Server', 's390x', source=True, basename='Server-s390x-'),
self._call('Server', 'x86_64', source=True, basename='Server-x86_64-')]
)