almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

gather: Create devel module for each normal module

Browse Source 
The module has same S:V:C, but the name is suffixed with `-devel`. The
module should contain all packages from the module koji tag that were
not included in the actual module.

The devel module has the same dependencies as the regular module, but
also additionally depends on the original module. The API and profiles
are cleared in the new module.

In the metadata it shows the same koji tag.

The test if package goes to the module is refactored to a function to
make work with the negated case a bit easier.

There may be unneeded multilib packages in the -devel module, because
there might be buildtime dependencies between things that we don't see.

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

Clear API and profiles

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2018-10-01 15:38:14 +02:00
parent 410d0125bc
commit 08d65bdde6
3 changed files with 106 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

143
pungi/phases/gather/sources/source_module.py
View File

@ -60,21 +60,49 @@ class GatherSourceModule(pungi.phases.gather.source.GatherSourceBase):
# store per-architecture artifacts there later.
variant.arch_mmds.setdefault(arch, {})
for mmd in variant.mmds:
mmd_id = "%s:%s:%s:%s" % (
nsvc = "%s:%s:%s:%s" % (
mmd.peek_name(),
mmd.peek_stream(),
mmd.peek_version(),
mmd.peek_context(),
)
if mmd_id not in variant.arch_mmds[arch]:
if nsvc not in variant.arch_mmds[arch]:
arch_mmd = mmd.copy()
variant.arch_mmds[arch][mmd_id] = arch_mmd
variant.arch_mmds[arch][nsvc] = arch_mmd
devel_nsvc = "%s-devel:%s:%s:%s" % (
mmd.peek_name(),
mmd.peek_stream(),
mmd.peek_version(),
mmd.peek_context(),
)
if devel_nsvc not in variant.arch_mmds[arch]:
arch_mmd = mmd.copy()
arch_mmd.set_name(arch_mmd.peek_name() + "-devel")
# Depend on the actual module
for dep in arch_mmd.get_dependencies():
dep.add_requires_single(mmd.peek_name(), mmd.peek_stream())
# Delete API and profiles
arch_mmd.set_rpm_api(Modulemd.SimpleSet())
arch_mmd.clear_profiles()
# Store the new modulemd
variant.arch_mmds[arch][devel_nsvc] = arch_mmd
variant.module_uid_to_koji_tag[devel_nsvc] = variant.module_uid_to_koji_tag.get(nsvc)
# Contains per-module RPMs added to variant.
added_rpms = {}
for mmd in variant.mmds:
nsvc = "%s:%s:%s:%s" % (
mmd.peek_name(),
mmd.peek_stream(),
mmd.peek_version(),
mmd.peek_context(),
)
arch_mmd = variant.arch_mmds[arch][nsvc]
rpms = sum([
variant.pkgset.rpms_by_arch.get(a, [])
variant.nsvc_to_pkgset[nsvc].rpms_by_arch.get(a, [])
for a in compatible_arches
], [])
for rpm_obj in rpms:
@ -85,59 +113,23 @@ class GatherSourceModule(pungi.phases.gather.source.GatherSourceBase):
log.write('Skipping %s due to incompatible arch\n' % rpm_obj)
continue
for mmd in variant.mmds:
mmd_id = "%s:%s:%s:%s" % (
if should_include(rpm_obj, arch, arch_mmd, multilib_arches):
# Add RPM to packages.
packages.add((rpm_obj, None))
added_rpms.setdefault(nsvc, [])
added_rpms[nsvc].append(str(rpm_obj.nevra))
log.write('Adding %s because it is in %s\n'
% (rpm_obj, nsvc))
else:
nsvc_devel = "%s-devel:%s:%s:%s" % (
mmd.peek_name(),
mmd.peek_stream(),
mmd.peek_version(),
mmd.peek_context(),
)
arch_mmd = variant.arch_mmds[arch][mmd_id]
srpm = kobo.rpmlib.parse_nvr(rpm_obj.sourcerpm)["name"]
filtered = False
buildopts = mmd.get_buildopts()
if buildopts:
whitelist = buildopts.get_rpm_whitelist()
if whitelist:
# We have whitelist, no filtering against components.
filtered = True
if srpm not in whitelist:
# Package is not on the list, skip it.
continue
if not filtered:
# Skip this mmd if this RPM does not belong to it.
if (srpm not in mmd.get_rpm_components().keys() or
rpm_obj.nevra not in mmd.get_rpm_artifacts().get()):
continue
# Filter out the RPM from artifacts if its filtered in MMD.
if rpm_obj.name in mmd.get_rpm_filter().get():
# No need to check if the rpm_obj is in rpm artifacts,
# the .remove() method does that anyway.
arch_mmd.get_rpm_artifacts().remove(str(rpm_obj.nevra))
continue
# Skip the rpm_obj if it's built for multilib arch, but
# multilib is not enabled for this srpm in MMD.
try:
mmd_component = mmd.get_rpm_components()[srpm]
multilib = mmd_component.get_multilib()
multilib = multilib.get() if multilib else set()
if arch not in multilib and rpm_obj.arch in multilib_arches:
continue
except KeyError:
# No such component, disable any multilib
if rpm_obj.arch not in ("noarch", arch):
continue
# Add RPM to packages.
packages.add((rpm_obj, None))
added_rpms.setdefault(mmd_id, [])
added_rpms[mmd_id].append(str(rpm_obj.nevra))
log.write('Adding %s because it is in %s\n'
% (rpm_obj, mmd_id))
added_rpms.setdefault(nsvc_devel, [])
added_rpms[nsvc_devel].append(str(rpm_obj.nevra))
log.write("Adding %s to %s module\n" % (rpm_obj, nsvc_devel))
# GatherSource returns all the packages in variant and does not
# care which package is in which module, but for modular metadata
@ -147,8 +139,8 @@ class GatherSourceModule(pungi.phases.gather.source.GatherSourceBase):
# particular module and use them to filter out the packages which
# have not been added to variant from the `arch_mmd`. This package
# list is later used in createrepo phase to generated modules.yaml.
for mmd_id, rpm_nevras in added_rpms.items():
arch_mmd = variant.arch_mmds[arch][mmd_id]
for nsvc, rpm_nevras in added_rpms.items():
arch_mmd = variant.arch_mmds[arch][nsvc]
artifacts = arch_mmd.get_rpm_artifacts()
# Modules without artifacts are also valid.
@ -162,3 +154,46 @@ class GatherSourceModule(pungi.phases.gather.source.GatherSourceBase):
arch_mmd.set_rpm_artifacts(added_artifacts)
return packages, groups
def should_include(rpm_obj, arch, arch_mmd, multilib_arches):
srpm = kobo.rpmlib.parse_nvr(rpm_obj.sourcerpm)["name"]
filtered = False
buildopts = arch_mmd.get_buildopts()
if buildopts:
whitelist = buildopts.get_rpm_whitelist()
if whitelist:
# We have whitelist, no filtering against components.
filtered = True
if srpm not in whitelist:
# Package is not on the list, skip it.
return False
if not filtered:
# Skip this mmd if this RPM does not belong to it.
if (srpm not in arch_mmd.get_rpm_components().keys() or
rpm_obj.nevra not in arch_mmd.get_rpm_artifacts().get()):
return False
# Filter out the RPM from artifacts if its filtered in MMD.
if rpm_obj.name in arch_mmd.get_rpm_filter().get():
# No need to check if the rpm_obj is in rpm artifacts,
# the .remove() method does that anyway.
arch_mmd.get_rpm_artifacts().remove(str(rpm_obj.nevra))
return False
# Skip the rpm_obj if it's built for multilib arch, but multilib is not
# enabled for this srpm in MMD.
try:
mmd_component = arch_mmd.get_rpm_components()[srpm]
multilib = mmd_component.get_multilib()
multilib = multilib.get() if multilib else set()
if arch not in multilib and rpm_obj.arch in multilib_arches:
return False
except KeyError:
# No such component, disable any multilib
if rpm_obj.arch not in ("noarch", arch):
return False
return True

3
tests/helpers.py
View File

@ -12,6 +12,7 @@ import errno
import imp
import six
from kobo.rpmlib import parse_nvr
from collections import defaultdict
from pungi.util import get_arch_variant_data
from pungi import paths, checks, Modulemd
@ -43,7 +44,7 @@ class MockVariant(mock.Mock):
self.pkgset = mock.Mock(rpms_by_arch={})
self.modules = None
self.name = name
self.nsvc_to_pkgset = {}
self.nsvc_to_pkgset = defaultdict(lambda: mock.Mock(rpms_by_arch={}))
def __str__(self):
return self.uid

4
tests/test_gather_source_module.py
View File

@ -30,13 +30,13 @@ class TestGatherSourceModule(helpers.PungiTestCase):
sourcerpm='pkg-1.0.0-1', nevra='pkg-0:1.0.0-1.x86_64',
arch="x86_64")
mock_rpm.name = 'pkg'
self.compose.variants['Server'].pkgset.rpms_by_arch['x86_64'] = [mock_rpm]
self.compose.variants['Server'].nsvc_to_pkgset["testmodule:master:1:2017"].rpms_by_arch['x86_64'] = [mock_rpm]
mock_rpm = mock.Mock(version='1.0.0', release='1',
epoch=0, excludearch=None, exclusivearch=None,
sourcerpm='pkg-1.0.0-1', nevra='pkg-0:1.0.0-1.i686',
arch="i686")
mock_rpm.name = 'pkg'
self.compose.variants['Server'].pkgset.rpms_by_arch['i686'] = [mock_rpm]
self.compose.variants['Server'].nsvc_to_pkgset["testmodule:master:1:2017"].rpms_by_arch['i686'] = [mock_rpm]
def test_gather_module(self):
source = GatherSourceModule(self.compose)