almalinux
/
pungi
15
1
Fork 0
Code Issues Pull Requests Releases Activity

Add support for modular composes 

Signed-off-by: Jan Kaluza <jkaluza@redhat.com>
This commit is contained in:
Jan Kaluza 2017-02-28 14:03:36 +01:00
parent 4b90822115
commit 9fcd71f831
9 changed files with 273 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
pungi/checks.py
View File

@ -493,7 +493,7 @@ def _make_schema():
},
"gather_source": {
"type": "string",
"enum": ["json", "comps", "none"],
"enum": ["module", "json", "comps", "none"],
},
"gather_fulltree": {
"type": "boolean",
@ -614,6 +614,10 @@ def _make_schema():
"global_target": {"type": "string"},
"global_release": {"$ref": "#/definitions/optional_string"},
"pdc_url": {"type": "string"},
"pdc_develop": {"type": "boolean", "default": False},
"pdc_insecure": {"type": "boolean", "default": False},
"koji_profile": {"type": "string"},
"pkgset_koji_tag": {"type": "string"},

18
pungi/phases/createrepo.py
View File

@ -181,6 +181,24 @@ def create_variant_repo(compose, arch, variant, pkg_type):
# this is a HACK to make CDN happy (dmach: at least I think, need to confirm with dgregor)
shutil.copy2(product_id_path, os.path.join(repo_dir, "repodata", "productid"))
# call modifyrepo to inject modulemd if needed
if variant.mmds:
import yaml
modules = {"modules": []}
for mmd in variant.mmds:
modules["modules"].append(mmd.dumps())
tmp_dir = compose.mkdtemp(prefix="pungi_")
modules_path = os.path.join(tmp_dir, "modules.yaml")
with open(modules_path, "w") as outfile:
outfile.write(yaml.safe_dump(modules))
cmd = repo.get_modifyrepo_cmd(os.path.join(repo_dir, "repodata"),
modules_path, mdtype="modules",
compress_type="gz")
log_file = compose.paths.log.log_file(
arch, "modifyrepo-modules-%s" % variant)
run(cmd, logfile=log_file, show_cmd=True)
shutil.rmtree(tmp_dir)
compose.log_info("[DONE ] %s" % msg)

44
pungi/phases/gather/sources/source_module.py Normal file
View File

@ -0,0 +1,44 @@
# -*- coding: utf-8 -*-
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <https://gnu.org/licenses/>.
"""
Get a package list based on modulemd metadata loaded in pkgset phase.
"""
from pungi.wrappers.comps import CompsWrapper
import pungi.phases.gather.source
import kobo.rpmlib
class GatherSourceModule(pungi.phases.gather.source.GatherSourceBase):
enabled = True
def __call__(self, arch, variant):
groups = set()
packages = set()
if variant is not None and variant.modules:
rpms = variant.pkgset.rpms_by_arch[arch] + \
variant.pkgset.rpms_by_arch["noarch"]
for rpm_obj in rpms:
for mmd in variant.mmds:
srpm = kobo.rpmlib.parse_nvr(rpm_obj.sourcerpm)["name"]
if (srpm in mmd.components.rpms.keys() and
rpm_obj.name not in mmd.filter.rpms):
packages.add((rpm_obj.name, None))
return packages, groups

9
pungi/phases/pkgset/pkgsets.py
View File

@ -148,15 +148,18 @@ class PackageSetBase(kobo.log.LoggingBase):
seen_sourcerpms = set()
# {Exclude,Exclusive}Arch must match *tree* arch + compatible native
# arches (excluding multilib arches)
exclusivearch_list = get_valid_arches(primary_arch, multilib=False,
add_noarch=False, add_src=False)
if primary_arch:
exclusivearch_list = get_valid_arches(
primary_arch, multilib=False, add_noarch=False, add_src=False)
else:
exclusivearch_list = None
for arch in arch_list:
self.rpms_by_arch.setdefault(arch, [])
for i in other.rpms_by_arch.get(arch, []):
if i.file_path in self.file_cache:
# TODO: test if it really works
continue
if arch == "noarch":
if exclusivearch_list and arch == "noarch":
if i.excludearch and set(i.excludearch) & set(exclusivearch_list):
self.log_debug("Excluding (EXCLUDEARCH: %s): %s"
% (sorted(set(i.excludearch)), i.file_name))

185
pungi/phases/pkgset/sources/source_koji.py
View File

@ -28,6 +28,84 @@ from pungi.phases.pkgset.common import create_arch_repos, create_global_repo, po
import pungi.phases.pkgset.source
try:
from pdc_client import PDCClient
import modulemd
WITH_MODULES = True
except:
WITH_MODULES = False
def get_pdc_client_session(compose):
if not WITH_MODULES:
compose.log_warning("pdc_client module is not installed, "
"support for modules is disabled")
return None
try:
return PDCClient(
server=compose.conf['pdc_url'],
develop=compose.conf['pdc_develop'],
ssl_verify=not compose.conf['pdc_insecure'],
)
except KeyError:
return None
def variant_dict_from_str(module_str):
module_info = {}
release_start = module_str.rfind('-')
module_info['variant_version'] = module_str[release_start+1:]
module_info['variant_id'] = module_str[:release_start]
module_info['variant_type'] = 'module'
return module_info
def get_module(session, module_info, strict=False):
"""
:param session : PDCClient instance
:param module_info: pdc variant_dict, str, mmd or module dict
:param strict: Normally this function returns None if no module can be
found. If strict=True, then a ValueError is raised.
:return final list of module_info which pass repoclosure
"""
module_info = variant_dict_from_str(module_info)
query = dict(
variant_id=module_info['variant_id'],
variant_version=module_info['variant_version'],
)
if module_info.get('variant_release'):
query['variant_release'] = module_info['variant_release']
retval = session['unreleasedvariants'](page_size=-1, **query)
# Error handling
if not retval:
if strict:
raise ValueError("Failed to find module in PDC %r" % query)
else:
return None
module = None
# If we specify 'variant_release', we expect only single module to be
# returned, but otherwise we have to pick the one with the highest
# release ourselves.
if 'variant_release' in query:
assert len(retval) <= 1, compose.log_error(
"More than one module returned from PDC: %s" % str(retval))
module = retval[0]
else:
module = retval[0]
for m in retval:
if int(m['variant_release']) > int(module['variant_release']):
module = m
return module
class PkgsetSourceKoji(pungi.phases.pkgset.source.PkgsetSourceBase):
enabled = True
@ -44,9 +122,7 @@ class PkgsetSourceKoji(pungi.phases.pkgset.source.PkgsetSourceBase):
def get_pkgset_from_koji(compose, koji_wrapper, path_prefix):
event_info = get_koji_event_info(compose, koji_wrapper)
tag_info = get_koji_tag_info(compose, koji_wrapper)
pkgset_global = populate_global_pkgset(compose, koji_wrapper, path_prefix, tag_info, event_info)
pkgset_global = populate_global_pkgset(compose, koji_wrapper, path_prefix, event_info)
package_sets = populate_arch_pkgsets(compose, path_prefix, pkgset_global)
package_sets["global"] = pkgset_global
@ -58,32 +134,90 @@ def get_pkgset_from_koji(compose, koji_wrapper, path_prefix):
return package_sets
def populate_global_pkgset(compose, koji_wrapper, path_prefix, compose_tag, event_id):
def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
all_arches = set(["src"])
for arch in compose.get_arches():
is_multilib = is_arch_multilib(compose.conf, arch)
arches = get_valid_arches(arch, is_multilib)
all_arches.update(arches)
compose_tag = compose.conf["pkgset_koji_tag"]
# List of compose tags from which we create this compose
compose_tags = []
# List of compose_tags per variant
variant_tags = {}
session = get_pdc_client_session(compose)
for variant in compose.all_variants.values():
variant.pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
koji_wrapper, compose.conf["sigkeys"], logger=compose._logger,
arches=all_arches)
variant_tags[variant] = []
# Find out all modules in every variant and add their compose tags
# to compose_tags list.
if session:
for module in variant.get_modules():
pdc_module = get_module(session, module["name"])
mmd = modulemd.ModuleMetadata()
mmd.loads(pdc_module["modulemd"])
tag = pdc_module["koji_tag"]
variant.mmds.append(mmd)
variant_tags[variant].append(tag)
if tag not in compose_tags:
compose_tags.append(tag)
if not variant_tags[variant]:
variant_tags[variant].append(compose.conf["pkgset_koji_tag"])
# In case we have no compose tag from module, use the default
# one from config.
if not compose_tags:
compose_tags.append(compose.conf["pkgset_koji_tag"])
inherit = compose.conf["pkgset_koji_inherit"]
msg = "Populating the global package set from tag '%s'" % compose_tag
global_pkgset_path = os.path.join(compose.paths.work.topdir(arch="global"), "pkgset_global.pickle")
global_pkgset_path = os.path.join(
compose.paths.work.topdir(arch="global"), "pkgset_global.pickle")
if compose.DEBUG and os.path.isfile(global_pkgset_path):
msg = "Populating the global package set from tag '%s'" % compose_tags
compose.log_warning("[SKIP ] %s" % msg)
pkgset = pickle.load(open(global_pkgset_path, "r"))
global_pkgset = pickle.load(open(global_pkgset_path, "r"))
else:
compose.log_info(msg)
pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(koji_wrapper, compose.conf["sigkeys"],
logger=compose._logger, arches=all_arches)
pkgset.populate(compose_tag, event_id, inherit=inherit)
global_pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
koji_wrapper, compose.conf["sigkeys"], logger=compose._logger,
arches=all_arches)
# Get package set for each compose tag and merge it to global package
# list. Also prepare per-variant pkgset, because we do not have list
# of binary RPMs in module definition - there is just list of SRPMs.
for compose_tag in compose_tags:
compose.log_info("Populating the global package set from tag "
"'%s'" % compose_tag)
pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
koji_wrapper, compose.conf["sigkeys"], logger=compose._logger,
arches=all_arches)
pkgset.populate(compose_tag, event_id, inherit=inherit)
for variant in compose.all_variants.values():
if compose_tag in variant_tags[variant]:
# Optimization for case where we have just single compose
# tag - we do not have to merge in this case...
if len(compose_tags) == 1:
variant.pkgset = pkgset
else:
variant.pkgset.merge(pkgset, None, list(all_arches))
# Optimization for case where we have just single compose
# tag - we do not have to merge in this case...
if len(compose_tags) == 1:
global_pkgset = pkgset
else:
global_pkgset.merge(pkgset, None, list(all_arches))
with open(global_pkgset_path, 'w') as f:
f.write(pickle.dumps(pkgset))
f.write(pickle.dumps(global_pkgset))
# write global package list
pkgset.save_file_list(compose.paths.work.package_list(arch="global"),
remove_path_prefix=path_prefix)
return pkgset
global_pkgset.save_file_list(
compose.paths.work.package_list(arch="global"),
remove_path_prefix=path_prefix)
return global_pkgset
def get_koji_event_info(compose, koji_wrapper):
@ -106,22 +240,3 @@ def get_koji_event_info(compose, koji_wrapper):
json.dump(result, open(event_file, "w"))
compose.log_info("Koji event: %s" % result["id"])
return result
def get_koji_tag_info(compose, koji_wrapper):
koji_proxy = koji_wrapper.koji_proxy
tag_file = os.path.join(compose.paths.work.topdir(arch="global"), "koji-tag")
msg = "Getting a koji tag info"
if compose.DEBUG and os.path.exists(tag_file):
compose.log_warning("[SKIP ] %s" % msg)
result = json.load(open(tag_file, "r"))
else:
compose.log_info(msg)
tag_name = compose.conf["pkgset_koji_tag"]
result = koji_proxy.getTag(tag_name)
if result is None:
raise ValueError("Unknown koji tag: %s" % tag_name)
result["name"] = tag_name
json.dump(result, open(tag_file, "w"))
compose.log_info("Koji compose tag: %(name)s (ID: %(id)s)" % result)
return result

37
pungi/wrappers/variants.py
View File

@ -77,6 +77,7 @@ class VariantsXmlParser(object):
"type": str(variant_node.attrib["type"]),
"arches": [str(i) for i in variant_node.xpath("arches/arch/text()")],
"groups": [],
"modules": [],
"environments": [],
"buildinstallpackages": [],
"is_empty": bool(variant_node.attrib.get("is_empty", False)),
@ -108,6 +109,15 @@ class VariantsXmlParser(object):
variant_dict["groups"].append(group)
for modulelist_node in variant_node.xpath("modules"):
for module_node in modulelist_node.xpath("module"):
module = {
"name": str(module_node.text),
"glob": self._is_true(module_node.attrib.get("glob", "false"))
}
variant_dict["modules"].append(module)
for environments_node in variant_node.xpath("environments"):
for environment_node in environments_node.xpath("environment"):
environment = {
@ -198,9 +208,11 @@ class VariantsXmlParser(object):
class Variant(object):
def __init__(self, id, name, type, arches, groups, environments=None,
buildinstallpackages=None, is_empty=False, parent=None):
buildinstallpackages=None, is_empty=False, parent=None,
modules=None):
environments = environments or []
modules = modules or []
buildinstallpackages = buildinstallpackages or []
self.id = id
@ -209,11 +221,16 @@ class Variant(object):
self.arches = sorted(copy.deepcopy(arches))
self.groups = sorted(copy.deepcopy(groups), lambda x, y: cmp(x["name"], y["name"]))
self.environments = sorted(copy.deepcopy(environments), lambda x, y: cmp(x["name"], y["name"]))
self.modules = sorted(copy.deepcopy(modules),
lambda x, y: cmp(x["name"], y["name"]))
self.buildinstallpackages = sorted(buildinstallpackages)
self.variants = {}
self.parent = parent
self.is_empty = is_empty
self.pkgset = None
self.mmds = []
def __getitem__(self, name):
return self.variants[name]
@ -274,6 +291,22 @@ class Variant(object):
result.append(group)
return result
def get_modules(self, arch=None, types=None, recursive=False):
"""Return list of groups, default types is ["self"]"""
types = types or ["self"]
result = copy.deepcopy(self.modules)
for variant in self.get_variants(arch=arch, types=types,
recursive=recursive):
if variant == self:
# XXX
continue
for module in variant.get_modules(arch=arch, types=types,
recursive=recursive):
if module not in result:
result.append(module)
return result
def get_variants(self, arch=None, types=None, recursive=False):
"""
Return all variants of given arch and types.
@ -339,6 +372,8 @@ def main(argv):
print(" ARCHES: %s" % ", ".join(sorted(i.arches)))
for group in i.groups:
print(" GROUP: %(name)-40s GLOB: %(glob)-5s DEFAULT: %(default)-5s USERVISIBLE: %(uservisible)-5s" % group)
for module in i.modules:
print(" MODULE: %(name)-40s GLOB: %(glob)-5s DEFAULT: %(default)-5s USERVISIBLE: %(uservisible)-5s" % module)
for env in i.environments:
print(" ENV: %(name)-40s DISPLAY_ORDER: %(display_order)s" % env)
print()

9
share/variants.dtd
View File

@ -1,6 +1,6 @@
<!ELEMENT variants (ref*,variant*)>
<!ELEMENT variant (release?,arches,groups?,environments*,variants*,buildinstallpackages?)?>
<!ELEMENT variant (release?,arches,groups?,environments*,variants*,buildinstallpackages?,modules?)?>
<!ATTLIST variant
id CDATA #REQUIRED
name CDATA #REQUIRED
@ -27,6 +27,13 @@
uservisible (true|false) #IMPLIED
>
<!ELEMENT modules (module)+>
<!ELEMENT module (#PCDATA)>
<!ATTLIST module
glob (true|false) #IMPLIED
>
<!ELEMENT environments (environment)+>
<!ELEMENT environment (#PCDATA)>

1
tests/helpers.py
View File

@ -33,6 +33,7 @@ class MockVariant(mock.Mock):
def __init__(self, *args, **kwargs):
super(MockVariant, self).__init__(*args, **kwargs)
self.parent = kwargs.get('parent', None)
self.mmds = []
def __str__(self):
return self.uid

54
tests/test_pkgset_source_koji.py
View File

@ -81,50 +81,6 @@ class TestGetKojiEvent(helpers.PungiTestCase):
with open(self.event_file) as f:
self.assertEqual(json.load(f), EVENT_INFO)
class TestGetKojiTagInfo(helpers.PungiTestCase):
def setUp(self):
super(TestGetKojiTagInfo, self).setUp()
self.compose = helpers.DummyCompose(self.topdir, {
'pkgset_koji_tag': 'f25'
})
self.compose.DEBUG = False
self.tag_file = os.path.join(self.topdir, 'work', 'global', 'koji-tag')
self.koji_wrapper = mock.Mock()
def test_get_tag_info(self):
self.koji_wrapper.koji_proxy.getTag.return_value = TAG_INFO
tag_info = source_koji.get_koji_tag_info(self.compose, self.koji_wrapper)
self.assertEqual(tag_info, TAG_INFO)
self.assertItemsEqual(
self.koji_wrapper.mock_calls,
[mock.call.koji_proxy.getTag('f25')]
)
with open(self.tag_file) as f:
self.assertEqual(json.load(f), TAG_INFO)
def test_bad_tag_name(self):
self.koji_wrapper.koji_proxy.getTag.return_value = None
with self.assertRaises(ValueError) as ctx:
source_koji.get_koji_tag_info(self.compose, self.koji_wrapper)
self.assertIn('Unknown', str(ctx.exception))
def test_get_tag_info_in_debug_mode(self):
self.compose.DEBUG = True
helpers.touch(self.tag_file, json.dumps(TAG_INFO))
tag_info = source_koji.get_koji_tag_info(self.compose, self.koji_wrapper)
self.assertEqual(tag_info, TAG_INFO)
self.assertItemsEqual(self.koji_wrapper.mock_calls, [])
with open(self.tag_file) as f:
self.assertEqual(json.load(f), TAG_INFO)
class TestPopulateGlobalPkgset(helpers.PungiTestCase):
def setUp(self):
super(TestPopulateGlobalPkgset, self).setUp()
@ -145,7 +101,7 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase):
orig_pkgset = KojiPackageSet.return_value
pkgset = source_koji.populate_global_pkgset(
self.compose, self.koji_wrapper, '/prefix', 'f25', 123456)
self.compose, self.koji_wrapper, '/prefix', 123456)
self.assertIs(pkgset, orig_pkgset)
self.assertEqual(
@ -168,7 +124,7 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase):
with mock.patch('pungi.phases.pkgset.sources.source_koji.open',
mock.mock_open(), create=True) as m:
pkgset = source_koji.populate_global_pkgset(
self.compose, self.koji_wrapper, '/prefix', 'f25', 123456)
self.compose, self.koji_wrapper, '/prefix', 123456)
self.assertEqual(pickle_load.call_args_list,
[mock.call(m.return_value)])
@ -204,12 +160,12 @@ class TestGetPackageSetFromKoji(helpers.PungiTestCase):
self.assertItemsEqual(
self.koji_wrapper.koji_proxy.mock_calls,
[mock.call.getLastEvent(),
mock.call.getTag('f25')]
[mock.call.getLastEvent()]
)
self.assertEqual(pgp.call_args_list,
[mock.call(self.compose, self.koji_wrapper, '/prefix',
TAG_INFO, EVENT_INFO)])
EVENT_INFO)])
self.assertEqual(pap.call_args_list,
[mock.call(self.compose, '/prefix', pgp.return_value)])
self.assertEqual(cgr.call_args_list,