diff --git a/fifloader.py b/fifloader.py index 8c6e8947..b62feadd 100755 --- a/fifloader.py +++ b/fifloader.py @@ -136,6 +136,7 @@ def merge_inputs(inputs, validate=False, clean=False): flavors = {} products = {} profiles = {} + pgroups = {} testsuites = {} jobtemplates = [] @@ -165,6 +166,7 @@ def merge_inputs(inputs, validate=False, clean=False): ('Flavors', flavors), ('Products', products), ('Profiles', profiles), + ('ProfileGroups', pgroups), ('JobTemplates', jobtemplates), ): if datatype in data: @@ -177,14 +179,22 @@ def merge_inputs(inputs, validate=False, clean=False): for (name, newsuite) in data['TestSuites'].items(): try: existing = testsuites[name] - # combine and stash the profiles - existing['profiles'].update(newsuite['profiles']) - combinedprofiles = existing['profiles'] + # combine and stash the profiles and groups + combinedprofiles = {} + if 'profiles' in existing: + existing['profiles'].update(newsuite['profiles']) + combinedprofiles = existing['profiles'] + combinedpgroups = {} + if 'profile_groups' in existing: + existing['profile_groups'].update(newsuite.get('profile_groups', {})) + combinedpgroups = existing['profile_groups'] # now update the existing suite with the new one, this - # will overwrite the profiles + # will overwrite the profiles and groups existing.update(newsuite) - # now restore the combined profiles + # now restore the combined profiles and groups existing['profiles'] = combinedprofiles + if combinedpgroups: + existing['profile_groups'] = combinedpgroups except KeyError: testsuites[name] = newsuite @@ -201,6 +211,8 @@ def merge_inputs(inputs, validate=False, clean=False): merged['Products'] = products if profiles: merged['Profiles'] = profiles + if pgroups: + merged['ProfileGroups'] = pgroups if testsuites: merged['TestSuites'] = testsuites if jobtemplates: @@ -211,19 +223,26 @@ def merge_inputs(inputs, validate=False, clean=False): schema_validate(merged, fif=True, state=state) print("Input template data is valid") - return (machines, flavors, products, profiles, testsuites, jobtemplates) + return (machines, flavors, products, profiles, pgroups, testsuites, jobtemplates) -def generate_job_templates(products, profiles, testsuites): +def generate_job_templates(products, profiles, pgroups, testsuites): """Given machines, products, profiles and testsuites (after merging and handling of flavors, but still in intermediate format), generates job templates and returns them as a list. """ jobtemplates = [] for (name, suite) in testsuites.items(): - if 'profiles' not in suite: + suiteprofs = {} + for (pgroup, baseprio) in suite.get('profile_groups', {}).items(): + if pgroup not in pgroups: + sys.exit(f"Error: profile group {pgroup} does not exist!") + for (gotprof, pprio) in pgroups[pgroup].items(): + suiteprofs[gotprof] = pprio+baseprio + suiteprofs.update(suite.get('profiles', {})) + if not suiteprofs: print("Warning: no profiles for test suite {}".format(name)) continue - for (profile, prio) in suite['profiles'].items(): + for (profile, prio) in suiteprofs.items(): jobtemplate = {'test_suite_name': name, 'prio': prio} # x86_64 compose jobtemplate['group_name'] = 'fedora' @@ -285,11 +304,13 @@ def reverse_qol(machines, flavors, products, testsuites): temp.update(product.get("settings", {})) product["settings"] = temp - # drop profiles from test suites - these are only used for job - # template generation and should not be in final output. if suite - # *only* contained profiles, drop it + # drop profiles and groups from test suites - these are only used + # for job template generation and should not be in final output. + # if suite *only* contained profiles/groups, drop it for suite in testsuites.values(): - del suite['profiles'] + for prop in ('profiles', 'profile_groups'): + if prop in suite: + del suite[prop] testsuites = {name: suite for (name, suite) in testsuites.items() if suite} machines = to_list_of_dicts(machines) @@ -343,9 +364,9 @@ def run(args): args = parse_args(args) if not args.validate and not args.write and not args.load: sys.exit("--no-validate specified and neither --write nor --load specified! Doing nothing.") - (machines, flavors, products, profiles, testsuites, jobtemplates) = merge_inputs( + (machines, flavors, products, profiles, pgroups, testsuites, jobtemplates) = merge_inputs( args.files, validate=args.validate, clean=args.clean) - jobtemplates.extend(generate_job_templates(products, profiles, testsuites)) + jobtemplates.extend(generate_job_templates(products, profiles, pgroups, testsuites)) (machines, products, testsuites) = reverse_qol(machines, flavors, products, testsuites) # now produce the output in upstream-compatible format out = {} diff --git a/schemas/fif-complete.json b/schemas/fif-complete.json index 75731871..ec362bfa 100644 --- a/schemas/fif-complete.json +++ b/schemas/fif-complete.json @@ -14,6 +14,7 @@ "Flavors": { "$ref": "fif-flavors.json" }, "ProductDefaults": { "$ref": "fif-productdefaults.json" }, "Products": { "$ref": "fif-products.json" }, + "ProfileGroups": { "$ref": "fif-profilegroups.json" }, "Profiles": { "$ref": "fif-profiles.json" }, "TestSuites": { "$ref": "fif-testsuites.json" }, "JobTemplates": { "$ref": "openqa-jobtemplates.json" } diff --git a/schemas/fif-incomplete.json b/schemas/fif-incomplete.json index 68dd2a81..4e80eec0 100644 --- a/schemas/fif-incomplete.json +++ b/schemas/fif-incomplete.json @@ -14,6 +14,7 @@ "Flavors": { "$ref": "fif-flavors.json" }, "ProductDefaults": { "$ref": "fif-productdefaults.json" }, "Products": { "$ref": "fif-products.json" }, + "ProfileGroups": { "$ref": "fif-profilegroups.json" }, "Profiles": { "$ref": "fif-profiles.json" }, "TestSuites": { "$ref": "fif-testsuites.json" }, "JobTemplates": { "$ref": "openqa-jobtemplates.json" } diff --git a/schemas/fif-predefault.json b/schemas/fif-predefault.json index f8ea7f60..d2fb08ef 100644 --- a/schemas/fif-predefault.json +++ b/schemas/fif-predefault.json @@ -14,6 +14,7 @@ "Flavors": { "$ref": "fif-flavors.json" }, "ProductDefaults": { "$ref": "fif-productdefaults.json" }, "Products": { "$ref": "fif-products-predefault.json" }, + "ProfileGroups": { "$ref": "fif-profilegroups.json" }, "Profiles": { "$ref": "fif-profiles.json" }, "TestSuites": { "$ref": "fif-testsuites.json" }, "JobTemplates": { "$ref": "openqa-jobtemplates.json" } diff --git a/schemas/fif-profilegroup.json b/schemas/fif-profilegroup.json new file mode 100644 index 00000000..2f1c26c5 --- /dev/null +++ b/schemas/fif-profilegroup.json @@ -0,0 +1,8 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "fif-profilegroup.json", + "title": "FIF single profile group schema", + "type": "object", + "minProperties": 1, + "additionalProperties": { "type": "number" } +} diff --git a/schemas/fif-profilegroups.json b/schemas/fif-profilegroups.json new file mode 100644 index 00000000..9f1cabe6 --- /dev/null +++ b/schemas/fif-profilegroups.json @@ -0,0 +1,8 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "fif-profilegroups.json", + "title": "FIF ProfileGroups object schema", + "type": "object", + "minProperties": 1, + "additionalProperties": { "$ref": "fif-profilegroup.json" } +} diff --git a/schemas/fif-testsuite.json b/schemas/fif-testsuite.json index e04a4ccd..6bdd40b5 100644 --- a/schemas/fif-testsuite.json +++ b/schemas/fif-testsuite.json @@ -2,8 +2,9 @@ "$schema": "http://json-schema.org/draft-07/schema#", "$id": "fif-testsuite.json", "title": "FIF single test suite schema", - "required": [ - "profiles" + "anyOf": [ + {"required": ["profiles"]}, + {"required": ["profile_groups"]} ], "properties": { "profiles": { @@ -11,6 +12,11 @@ "title": "A testsuite profile entry schema", "additionalProperties": { "type": "number" } }, + "profile_groups": { + "type": "object", + "title": "A profile group entry schema", + "additionalProperties": { "type": "number" } + }, "settings": { "$ref": "fif-settingshash.json" } }, "additionalProperties": false diff --git a/unittests/data/templates-updates.fif.json b/unittests/data/templates-updates.fif.json index 5d115710..b3ea7631 100644 --- a/unittests/data/templates-updates.fif.json +++ b/unittests/data/templates-updates.fif.json @@ -25,6 +25,11 @@ "product": "fedora-updates-server-x86_64-*" } }, + "ProfileGroups": { + "fedora-updates-server-1arch": { + "fedora-updates-server-ppc64le-*-ppc64le": 1 + } + }, "TestSuites": { "advisory_boot": { "profiles": { @@ -39,8 +44,10 @@ } }, "base_selinux": { + "profile_groups": { + "fedora-updates-server-1arch": 40 + }, "profiles": { - "fedora-updates-server-ppc64le-*-ppc64le": 40, "fedora-updates-server-x86_64-*-64bit": 40 } } diff --git a/unittests/data/templates.fif.json b/unittests/data/templates.fif.json index 62e384c9..afb41915 100644 --- a/unittests/data/templates.fif.json +++ b/unittests/data/templates.fif.json @@ -65,10 +65,21 @@ "product": "fedora-Server-dvd-iso-x86_64-*" } }, + "ProfileGroups": { + "fedora-server-2arch": { + "fedora-Server-dvd-iso-ppc64le-*-ppc64le": 0, + "fedora-Server-dvd-iso-x86_64-*-64bit": 1 + }, + "fedora-server-1arch": { + "fedora-Server-dvd-iso-ppc64le-*-ppc64le": 0 + } + }, "TestSuites": { "base_selinux": { + "profile_groups": { + "fedora-server-1arch": 40 + }, "profiles": { - "fedora-Server-dvd-iso-ppc64le-*-ppc64le": 40, "fedora-Server-dvd-iso-x86_64-*-64bit": 40 }, "settings": { @@ -81,9 +92,8 @@ } }, "install_default_upload": { - "profiles": { - "fedora-Server-dvd-iso-ppc64le-*-ppc64le": 10, - "fedora-Server-dvd-iso-x86_64-*-64bit": 10 + "profile_groups": { + "fedora-server-2arch": 10 }, "settings": { "PACKAGE_SET": "default", diff --git a/unittests/test_fifloader.py b/unittests/test_fifloader.py index 5a0b9128..1b23a665 100644 --- a/unittests/test_fifloader.py +++ b/unittests/test_fifloader.py @@ -73,20 +73,26 @@ def test_schema_validate(): ) def test_merge_inputs(input1, input2): """Test for merge_inputs.""" - (machines, flavors, products, profiles, testsuites, jobtemplates) = _get_merged(input1, input2) + (machines, flavors, products, profiles, pgroups, testsuites, jobtemplates) = _get_merged( + input1, + input2 + ) # a few known attributes of the test data to ensure the merge worked assert len(machines) == 2 assert len(flavors) == 1 assert len(products) == 4 assert len(profiles) == 4 + assert len(pgroups) == 3 assert not jobtemplates # testsuite merging is the most complex feature # len should be 3 as there is 1 unique suite in each input file, # and one defined in both which should be merged assert len(testsuites) == 3 # check the merged suite was merged correctly - # we should have the profiles from *both* input files... - assert len(testsuites['base_selinux']['profiles']) == 4 + # we should have the profiles and profile groups from *both* + # input files... + assert len(testsuites['base_selinux']['profiles']) == 2 + assert len(testsuites['base_selinux']['profile_groups']) == 2 # and we should still have the settings (note, combining settings # is not supported, the last-read settings dict is always used) assert len(testsuites['base_selinux']['settings']) == 6 @@ -98,8 +104,8 @@ def test_merge_inputs(input1, input2): def test_generate_job_templates(): """Test for generate_job_templates.""" - (machines, _, products, profiles, testsuites, _) = _get_merged() - templates = fifloader.generate_job_templates(products, profiles, testsuites) + (machines, _, products, profiles, pgroups, testsuites, _) = _get_merged() + templates = fifloader.generate_job_templates(products, profiles, pgroups, testsuites) # we should get one template per profile in merged input assert len(templates) == 8 for template in templates: @@ -112,9 +118,17 @@ def test_generate_job_templates(): assert item in template assert template['test_suite_name'] in list(testsuites.keys()) + # check profile group expansion + idus = [t for t in templates if t['test_suite_name'] == 'install_default_upload'] + assert len(idus) == 2 + assert {t['machine_name'] for t in idus} == {'ppc64le', '64bit'} + aboots = [t for t in templates if t['test_suite_name'] == 'base_selinux'] + assert len(aboots) == 4 + assert {t['machine_name'] for t in aboots} == {'ppc64le', '64bit'} + def test_reverse_qol(): """Test for reverse_qol.""" - (machines, flavors, products, _, testsuites, _) = _get_merged() + (machines, flavors, products, _, _, testsuites, _) = _get_merged() (machines, products, testsuites) = fifloader.reverse_qol(machines, flavors, products, testsuites) assert isinstance(machines, list) assert isinstance(products, list) @@ -127,8 +141,9 @@ def test_reverse_qol(): for item in datatype: # all items should have one of these settlists.append(item['settings']) - # no items should have one of these + # no items should have these assert 'profiles' not in item + assert 'profile_groups' not in item for settlist in settlists: assert isinstance(settlist, list) for setting in settlist: