almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

Write module metadata

Browse Source 
Signed-off-by: Ondrej Nosek <onosek@redhat.com>
This commit is contained in:
Ondrej Nosek 2018-03-20 08:52:49 +01:00
parent c6d507582a
commit e3aa2f769b
6 changed files with 124 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
pungi/phases/createrepo.py
View File

@ -19,12 +19,13 @@ __all__ = (
) )
import os
import glob
import shutil
import threading
import copy import copy
import errno import errno
import glob
import os
import shutil
import threading
import xml.dom.minidom
from kobo.threads import ThreadPool, WorkerThread from kobo.threads import ThreadPool, WorkerThread
from kobo.shortcuts import run, relative_path from kobo.shortcuts import run, relative_path
@ -36,6 +37,7 @@ from ..util import find_old_compose, temp_dir, get_arch_variant_data
from pungi import Modulemd from pungi import Modulemd
import productmd.rpms import productmd.rpms
import productmd.modules
createrepo_lock = threading.Lock() createrepo_lock = threading.Lock()
@ -48,6 +50,7 @@ class CreaterepoPhase(PhaseBase):
def __init__(self, compose): def __init__(self, compose):
PhaseBase.__init__(self, compose) PhaseBase.__init__(self, compose)
self.pool = ThreadPool(logger=self.compose._logger) self.pool = ThreadPool(logger=self.compose._logger)
self.modules_metadata = ModulesMetadata(compose)
def validate(self): def validate(self):
errors = [] errors = []
@ -70,22 +73,26 @@ class CreaterepoPhase(PhaseBase):
for variant in self.compose.get_variants(): for variant in self.compose.get_variants():
if variant.is_empty: if variant.is_empty:
continue continue
self.pool.queue_put((self.compose, None, variant, "srpm")) self.pool.queue_put((self.compose, None, variant, "srpm", self.modules_metadata))
for arch in variant.arches: for arch in variant.arches:
self.pool.queue_put((self.compose, arch, variant, "rpm")) self.pool.queue_put((self.compose, arch, variant, "rpm", self.modules_metadata))
self.pool.queue_put((self.compose, arch, variant, "debuginfo")) self.pool.queue_put((self.compose, arch, variant, "debuginfo", self.modules_metadata))
self.pool.start() self.pool.start()
def stop(self):
super(CreaterepoPhase, self).stop()
self.modules_metadata.write_modules_metadata()
def create_variant_repo(compose, arch, variant, pkg_type):
def create_variant_repo(compose, arch, variant, pkg_type, modules_metadata=None):
types = { types = {
'rpm': ('binary', 'rpm': ('binary',
lambda: compose.paths.compose.repository(arch=arch, variant=variant)), lambda **kwargs: compose.paths.compose.repository(arch=arch, variant=variant, **kwargs)),
'srpm': ('source', 'srpm': ('source',
lambda: compose.paths.compose.repository(arch='src', variant=variant)), lambda **kwargs: compose.paths.compose.repository(arch='src', variant=variant, **kwargs)),
'debuginfo': ('debug', 'debuginfo': ('debug',
lambda: compose.paths.compose.debug_repository(arch=arch, variant=variant)), lambda **kwargs: compose.paths.compose.debug_repository(arch=arch, variant=variant, **kwargs)),
} }
if variant.is_empty or (arch is None and pkg_type != 'srpm'): if variant.is_empty or (arch is None and pkg_type != 'srpm'):
@ -186,7 +193,8 @@ def create_variant_repo(compose, arch, variant, pkg_type):
# call modifyrepo to inject modulemd if needed # call modifyrepo to inject modulemd if needed
if arch in variant.arch_mmds and Modulemd is not None: if arch in variant.arch_mmds and Modulemd is not None:
modules = [] modules = []
for mmd in variant.arch_mmds[arch].values(): metadata = []
for module_id, mmd in variant.arch_mmds[arch].items():
# Create copy of architecture specific mmd to filter out packages # Create copy of architecture specific mmd to filter out packages
# which are not part of this particular repo. # which are not part of this particular repo.
repo_mmd = Modulemd.Module.new_from_string(mmd.dumps()) repo_mmd = Modulemd.Module.new_from_string(mmd.dumps())
@ -196,11 +204,18 @@ def create_variant_repo(compose, arch, variant, pkg_type):
if not artifacts or artifacts.size() == 0: if not artifacts or artifacts.size() == 0:
continue continue
module_rpms = set()
repo_artifacts = Modulemd.SimpleSet() repo_artifacts = Modulemd.SimpleSet()
for rpm_nevra in rpm_nevras: for rpm_nevra in rpm_nevras:
if artifacts.contains(rpm_nevra): if artifacts.contains(rpm_nevra):
repo_artifacts.add(rpm_nevra) repo_artifacts.add(rpm_nevra)
module_rpms.add(rpm_nevra)
repo_mmd.set_rpm_artifacts(repo_artifacts) repo_mmd.set_rpm_artifacts(repo_artifacts)
if module_rpms: # do not create metadata if there is empty rpm list
if modules_metadata: # some unittests call this method without parameter modules_metadata and its default is None
metadata.append((module_id, module_rpms))
else:
raise AttributeError("module_metadata parameter was not passed and it is needed for module processing")
modules.append(repo_mmd) modules.append(repo_mmd)
with temp_dir() as tmp_dir: with temp_dir() as tmp_dir:
@ -214,13 +229,26 @@ def create_variant_repo(compose, arch, variant, pkg_type):
arch, "modifyrepo-modules-%s" % variant) arch, "modifyrepo-modules-%s" % variant)
run(cmd, logfile=log_file, show_cmd=True) run(cmd, logfile=log_file, show_cmd=True)
for module_id, module_rpms in metadata:
modulemd_path = os.path.join(types[pkg_type][1](relative=True), find_file_in_repodata(repo_dir, 'modules'))
modules_metadata.prepare_module_metadata(variant, arch, module_id, modulemd_path, types[pkg_type][0], list(module_rpms))
compose.log_info("[DONE ] %s" % msg) compose.log_info("[DONE ] %s" % msg)
def find_file_in_repodata(repo_path, type_):
dom = xml.dom.minidom.parse(os.path.join(repo_path, 'repodata', 'repomd.xml'))
for entry in dom.getElementsByTagName('data'):
if entry.getAttribute('type') == type_:
return entry.getElementsByTagName('location')[0].getAttribute('href')
entry.unlink()
raise RuntimeError('No such file in repodata: %s' % type_)
class CreaterepoThread(WorkerThread): class CreaterepoThread(WorkerThread):
def process(self, item, num): def process(self, item, num):
compose, arch, variant, pkg_type = item compose, arch, variant, pkg_type, modules_metadata = item
create_variant_repo(compose, arch, variant, pkg_type=pkg_type) create_variant_repo(compose, arch, variant, pkg_type=pkg_type, modules_metadata=modules_metadata)
def get_productids_from_scm(compose): def get_productids_from_scm(compose):
@ -317,3 +345,33 @@ def _has_deltas(compose, variant, arch):
if isinstance(compose.conf.get(key), bool): if isinstance(compose.conf.get(key), bool):
return compose.conf[key] return compose.conf[key]
return any(get_arch_variant_data(compose.conf, key, arch, variant)) return any(get_arch_variant_data(compose.conf, key, arch, variant))
class ModulesMetadata(object):
def __init__(self, compose):
# Prepare empty module metadata
self.compose = compose
self.modules_metadata_file = self.compose.paths.compose.metadata("modules.json")
self.productmd_modules_metadata = productmd.modules.Modules()
self.productmd_modules_metadata.compose.id = copy.copy(self.compose.compose_id)
self.productmd_modules_metadata.compose.type = copy.copy(self.compose.compose_type)
self.productmd_modules_metadata.compose.date = copy.copy(self.compose.compose_date)
self.productmd_modules_metadata.compose.respin = copy.copy(self.compose.compose_respin)
def write_modules_metadata(self):
"""
flush modules metadata into file
"""
self.compose.log_info("Writing modules metadata: %s" % self.modules_metadata_file)
self.productmd_modules_metadata.dump(self.modules_metadata_file)
def prepare_module_metadata(self, variant, arch, module_id, modulemd_path, category, module_rpms):
"""
find uid/koji_tag which is correstponding with variant object and
add record(s) into module metadata structure
"""
for uid, koji_tag in variant.module_uid_to_koji_tag.items():
uid_dict = self.productmd_modules_metadata.parse_uid(uid)
if module_id == '{module_name}-{stream}'.format(**uid_dict):
self.productmd_modules_metadata.add(variant.uid, arch, uid, koji_tag, modulemd_path, category, module_rpms)
break

19
pungi/phases/pkgset/sources/source_koji.py
View File

@ -254,8 +254,13 @@ def _get_modules_from_pdc(compose, session, variant, variant_tags):
_add_module_to_variant(variant, mmd, pdc_module["rpms"]) _add_module_to_variant(variant, mmd, pdc_module["rpms"])
tag = pdc_module["koji_tag"] tag = pdc_module["koji_tag"]
uid = pdc_module["variant_uid"]
variant_tags[variant].append(tag) variant_tags[variant].append(tag)
# Store mapping module-uid --> koji_tag into variant.
# This is needed in createrepo phase where metadata is exposed by producmd
variant.module_uid_to_koji_tag[uid] = tag
module_msg = "Module {module} in variant {variant} will use Koji tag {tag}.".format( module_msg = "Module {module} in variant {variant} will use Koji tag {tag}.".format(
variant=variant, tag=tag, module=module["name"]) variant=variant, tag=tag, module=module["name"])
compose.log_info("%s" % module_msg) compose.log_info("%s" % module_msg)
@ -334,12 +339,24 @@ def _get_modules_from_koji_tags(
mmd.upgrade() mmd.upgrade()
_add_module_to_variant(variant, mmd, rpms, True) _add_module_to_variant(variant, mmd, rpms, True)
# Store mapping module-uid --> koji_tag into variant.
# This is needed in createrepo phase where metadata is exposed by producmd
module_data = build.get("extra", {}).get("typeinfo", {}).get("module", {})
try:
uid = "{name}:{stream}".format(**module_data)
except KeyError as e:
raise KeyError("Unable to create uid in format name:stream %s" % e)
if module_data.get("version"):
uid += ":{version}".format(**module_data)
if module_data.get("context"):
uid += ":{context}".format(**module_data)
variant.module_uid_to_koji_tag[uid] = module_tag
module_msg = "Module {module} in variant {variant} will use Koji tag {tag}.".format( module_msg = "Module {module} in variant {variant} will use Koji tag {tag}.".format(
variant=variant, tag=module_tag, module=build["nvr"]) variant=variant, tag=module_tag, module=build["nvr"])
compose.log_info("%s" % module_msg) compose.log_info("%s" % module_msg)
def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
all_arches = set(["src"]) all_arches = set(["src"])
for arch in compose.get_arches(): for arch in compose.get_arches():

1
pungi/wrappers/variants.py
View File

@ -236,6 +236,7 @@ class Variant(object):
self.pkgset = None self.pkgset = None
self.mmds = [] self.mmds = []
self.arch_mmds = {} self.arch_mmds = {}
self.module_uid_to_koji_tag = {}
def __getitem__(self, name): def __getitem__(self, name):
return self.variants[name] return self.variants[name]

1
tests/helpers.py
View File

@ -38,6 +38,7 @@ class MockVariant(mock.Mock):
self.parent = kwargs.get('parent', None) self.parent = kwargs.get('parent', None)
self.mmds = [] self.mmds = []
self.arch_mmds = {} self.arch_mmds = {}
self.module_uid_to_koji_tag = {}
self.variants = {} self.variants = {}
self.pkgset = mock.Mock(rpms_by_arch={}) self.pkgset = mock.Mock(rpms_by_arch={})
self.modules = None self.modules = None

57
tests/test_createrepophase.py
View File

@ -16,7 +16,8 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
from pungi.phases.createrepo import (CreaterepoPhase, from pungi.phases.createrepo import (CreaterepoPhase,
create_variant_repo, create_variant_repo,
get_productids_from_scm) get_productids_from_scm,
ModulesMetadata)
from tests.helpers import DummyCompose, PungiTestCase, copy_fixture, touch from tests.helpers import DummyCompose, PungiTestCase, copy_fixture, touch
from pungi import Modulemd from pungi import Modulemd
@ -75,19 +76,19 @@ class TestCreaterepoPhase(PungiTestCase):
self.assertEqual(len(pool.add.mock_calls), 5) self.assertEqual(len(pool.add.mock_calls), 5)
self.assertItemsEqual( self.assertItemsEqual(
pool.queue_put.mock_calls, pool.queue_put.mock_calls,
[mock.call((compose, 'x86_64', compose.variants['Server'], 'rpm')), [mock.call((compose, 'x86_64', compose.variants['Server'], 'rpm', phase.modules_metadata)),
mock.call((compose, 'x86_64', compose.variants['Server'], 'debuginfo')), mock.call((compose, 'x86_64', compose.variants['Server'], 'debuginfo', phase.modules_metadata)),
mock.call((compose, 'amd64', compose.variants['Server'], 'rpm')), mock.call((compose, 'amd64', compose.variants['Server'], 'rpm', phase.modules_metadata)),
mock.call((compose, 'amd64', compose.variants['Server'], 'debuginfo')), mock.call((compose, 'amd64', compose.variants['Server'], 'debuginfo', phase.modules_metadata)),
mock.call((compose, None, compose.variants['Server'], 'srpm')), mock.call((compose, None, compose.variants['Server'], 'srpm', phase.modules_metadata)),
mock.call((compose, 'x86_64', compose.variants['Everything'], 'rpm')), mock.call((compose, 'x86_64', compose.variants['Everything'], 'rpm', phase.modules_metadata)),
mock.call((compose, 'x86_64', compose.variants['Everything'], 'debuginfo')), mock.call((compose, 'x86_64', compose.variants['Everything'], 'debuginfo', phase.modules_metadata)),
mock.call((compose, 'amd64', compose.variants['Everything'], 'rpm')), mock.call((compose, 'amd64', compose.variants['Everything'], 'rpm', phase.modules_metadata)),
mock.call((compose, 'amd64', compose.variants['Everything'], 'debuginfo')), mock.call((compose, 'amd64', compose.variants['Everything'], 'debuginfo', phase.modules_metadata)),
mock.call((compose, None, compose.variants['Everything'], 'srpm')), mock.call((compose, None, compose.variants['Everything'], 'srpm', phase.modules_metadata)),
mock.call((compose, 'amd64', compose.variants['Client'], 'rpm')), mock.call((compose, 'amd64', compose.variants['Client'], 'rpm', phase.modules_metadata)),
mock.call((compose, 'amd64', compose.variants['Client'], 'debuginfo')), mock.call((compose, 'amd64', compose.variants['Client'], 'debuginfo', phase.modules_metadata)),
mock.call((compose, None, compose.variants['Client'], 'srpm'))]) mock.call((compose, None, compose.variants['Client'], 'srpm', phase.modules_metadata))])
@mock.patch('pungi.checks.get_num_cpus') @mock.patch('pungi.checks.get_num_cpus')
@mock.patch('pungi.phases.createrepo.ThreadPool') @mock.patch('pungi.phases.createrepo.ThreadPool')
@ -105,16 +106,16 @@ class TestCreaterepoPhase(PungiTestCase):
self.assertEqual(len(pool.add.mock_calls), 5) self.assertEqual(len(pool.add.mock_calls), 5)
self.assertItemsEqual( self.assertItemsEqual(
pool.queue_put.mock_calls, pool.queue_put.mock_calls,
[mock.call((compose, 'x86_64', compose.variants['Server'], 'rpm')), [mock.call((compose, 'x86_64', compose.variants['Server'], 'rpm', phase.modules_metadata)),
mock.call((compose, 'x86_64', compose.variants['Server'], 'debuginfo')), mock.call((compose, 'x86_64', compose.variants['Server'], 'debuginfo', phase.modules_metadata)),
mock.call((compose, 'amd64', compose.variants['Server'], 'rpm')), mock.call((compose, 'amd64', compose.variants['Server'], 'rpm', phase.modules_metadata)),
mock.call((compose, 'amd64', compose.variants['Server'], 'debuginfo')), mock.call((compose, 'amd64', compose.variants['Server'], 'debuginfo', phase.modules_metadata)),
mock.call((compose, None, compose.variants['Server'], 'srpm')), mock.call((compose, None, compose.variants['Server'], 'srpm', phase.modules_metadata)),
mock.call((compose, 'x86_64', compose.variants['Everything'], 'rpm')), mock.call((compose, 'x86_64', compose.variants['Everything'], 'rpm', phase.modules_metadata)),
mock.call((compose, 'x86_64', compose.variants['Everything'], 'debuginfo')), mock.call((compose, 'x86_64', compose.variants['Everything'], 'debuginfo', phase.modules_metadata)),
mock.call((compose, 'amd64', compose.variants['Everything'], 'rpm')), mock.call((compose, 'amd64', compose.variants['Everything'], 'rpm', phase.modules_metadata)),
mock.call((compose, 'amd64', compose.variants['Everything'], 'debuginfo')), mock.call((compose, 'amd64', compose.variants['Everything'], 'debuginfo', phase.modules_metadata)),
mock.call((compose, None, compose.variants['Everything'], 'srpm'))]) mock.call((compose, None, compose.variants['Everything'], 'srpm', phase.modules_metadata))])
class TestCreateVariantRepo(PungiTestCase): class TestCreateVariantRepo(PungiTestCase):
@ -750,10 +751,11 @@ class TestCreateVariantRepo(PungiTestCase):
[mock.call(repodata_dir, ANY, compress_type='gz', mdtype='modules')]) [mock.call(repodata_dir, ANY, compress_type='gz', mdtype='modules')])
@unittest.skipUnless(Modulemd is not None, 'Skipped test, no module support.') @unittest.skipUnless(Modulemd is not None, 'Skipped test, no module support.')
@mock.patch('pungi.phases.createrepo.find_file_in_repodata')
@mock.patch('pungi.phases.createrepo.run') @mock.patch('pungi.phases.createrepo.run')
@mock.patch('pungi.phases.createrepo.CreaterepoWrapper') @mock.patch('pungi.phases.createrepo.CreaterepoWrapper')
def test_variant_repo_modules_artifacts( def test_variant_repo_modules_artifacts(
self, CreaterepoWrapperCls, run): self, CreaterepoWrapperCls, run, modulemd_filename):
compose = DummyCompose(self.topdir, { compose = DummyCompose(self.topdir, {
'createrepo_checksum': 'sha256', 'createrepo_checksum': 'sha256',
}) })
@ -784,7 +786,10 @@ class TestCreateVariantRepo(PungiTestCase):
compose.paths.compose.os_tree('x86_64', compose.variants['Server']), compose.paths.compose.os_tree('x86_64', compose.variants['Server']),
'repodata') 'repodata')
create_variant_repo(compose, 'x86_64', compose.variants['Server'], 'rpm') modules_metadata = ModulesMetadata(compose)
modulemd_filename.return_value = "Server/x86_64/os/repodata/3511d16a723e1bd69826e591508f07e377d2212769b59178a9-modules.yaml.gz"
create_variant_repo(compose, 'x86_64', compose.variants['Server'], 'rpm', modules_metadata)
self.assertItemsEqual( self.assertItemsEqual(
repo.get_modifyrepo_cmd.mock_calls, repo.get_modifyrepo_cmd.mock_calls,

2
tests/test_pkgset_source_koji.py
View File

@ -142,7 +142,7 @@ data:
- MIT - MIT
""" """
get_module.return_value = {'abc': 'def', 'modulemd': modulemd, 'rpms': [], 'koji_tag': 'taggg'} get_module.return_value = {'abc': 'def', 'modulemd': modulemd, 'rpms': [], 'koji_tag': 'taggg', 'variant_uid': 'modulenamefoo-rhel-1'}
for name, variant in self.compose.variants.items(): for name, variant in self.compose.variants.items():
variant.get_modules = mock.MagicMock() variant.get_modules = mock.MagicMock()
if name == 'Server': if name == 'Server':