diff --git a/fifloader.py b/fifloader.py index 3b1eb43b..0ead63d8 100755 --- a/fifloader.py +++ b/fifloader.py @@ -25,9 +25,9 @@ on this format as it compares to the upstream format. It produces data in the up can write this data to a JSON file and/or call the upstream loader on it directly, depending on the command-line arguments specified. -The input data must contain definitions of Machines, Products, TestSuites, and Profiles. The input -data *may* contain JobTemplates, but does not have to and is expected to contain none or only a few -oddballs. +The input data must contain definitions of Machines, Products, TestSuites, and Profiles. It may +also contain Flavors. It also *may* contain JobTemplates, but does not have to and is expected to +contain none or only a few oddballs. The format for Machines, Products and TestSuites is based on the upstream format but with various quality-of-life improvements. Upstream, each of these is a list-of-dicts, each dict containing a @@ -36,7 +36,16 @@ easier to read and easier to access). In the upstream format, each Machine, Prod dict can contain an entry with the key 'settings' which defines variables. The value (for some reason...) is a list of dicts, each dict of the format {"key": keyname, "value": value}. This loader expects a more obvious and simple format where the value of the 'settings' key is simply a -dict of keys and values. +dict of keys and values. With this loader, Products can inherit settings from Flavors to reduce +duplication - see below. + +The expected format of the Flavors dict is a dict-of-dicts. For each entry, the key is a flavor +name that is expected to be used as the 'flavor' for one or more Product(s). The value is a dict +with only a 'settings' key, containing settings in the same format described above. When +processing Products, fifloader will merge in the settings from the product's flavor, if it +exists in the Flavors dict and defines any settings. If both the Product and the Flavor define +a given setting, the Product's definition wins. The purpose of the Flavors dict is to reduce +duplication of settings between multiple products with the same flavor. The expected format of the Profiles dict is a dict-of-dicts. For each entry, the key is a unique name, and the value is a dict with keys 'machine' and 'product', each value being a valid name from @@ -116,10 +125,12 @@ def schema_validate(instance, fif=True, complete=True, schemapath=SCHEMAPATH): def merge_inputs(inputs, validate=False, clean=False): """Merge multiple input files. Expects JSON file names. Optionally validates the input files before merging, and the merged output. - Returns a 5-tuple of machines, products, profiles, testsuites and - jobtemplates (the first four as dicts, the fifth as a list). + Returns a 6-tuple of machines, flavors, products, profiles, + testsuites and jobtemplates (the first four as dicts, the fifth as + a list). """ machines = {} + flavors = {} products = {} profiles = {} testsuites = {} @@ -141,6 +152,7 @@ def merge_inputs(inputs, validate=False, clean=False): # simple merges for all these for (datatype, tgt) in ( ('Machines', machines), + ('Flavors', flavors), ('Products', products), ('Profiles', profiles), ('JobTemplates', jobtemplates), @@ -173,6 +185,8 @@ def merge_inputs(inputs, validate=False, clean=False): merged = {} if machines: merged['Machines'] = machines + if flavors: + merged['Flavors'] = flavors if products: merged['Products'] = products if profiles: @@ -184,12 +198,12 @@ def merge_inputs(inputs, validate=False, clean=False): schema_validate(merged, fif=True, complete=clean) print("Input template data is valid") - return (machines, products, profiles, testsuites, jobtemplates) + return (machines, flavors, products, profiles, testsuites, jobtemplates) def generate_job_templates(products, profiles, testsuites): """Given machines, products, profiles and testsuites (after - merging, but still in intermediate format), generates job - templates and returns them as a list. + merging and handling of flavors, but still in intermediate format), + generates job templates and returns them as a list. """ jobtemplates = [] for (name, suite) in testsuites.items(): @@ -222,12 +236,13 @@ def generate_job_templates(products, profiles, testsuites): jobtemplates.append(jobtemplate) return jobtemplates -def reverse_qol(machines, products, testsuites): +def reverse_qol(machines, flavors, products, testsuites): """Reverse all our quality-of-life improvements in Machines, - Products and TestSuites. We don't do profiles as only this loader - uses them, upstream loader does not. We don't do jobtemplates as - we don't do any QOL stuff for that. Returns the same tuple it's - passed. + Flavors, Products and TestSuites. We don't do profiles as only + this loader uses them, upstream loader does not. We don't do + jobtemplates as we don't do any QOL stuff for that. Returns + machines, products and testsuites - flavors are a loader-only + concept. """ # first, some nested convenience functions def to_list_of_dicts(datadict): @@ -249,6 +264,14 @@ def reverse_qol(machines, products, testsuites): converted.append({'key': key, 'value': value}) return converted + # merge flavors into products + for product in products.values(): + flavsets = flavors.get(product["flavor"], {}).get("settings", {}) + if flavsets: + temp = dict(flavsets) + temp.update(product.get("settings", {})) + product["settings"] = temp + # drop profiles from test suites - these are only used for job # template generation and should not be in final output. if suite # *only* contained profiles, drop it @@ -307,10 +330,10 @@ def run(args): args = parse_args(args) if not args.validate and not args.write and not args.load: sys.exit("--no-validate specified and neither --write nor --load specified! Doing nothing.") - (machines, products, profiles, testsuites, jobtemplates) = merge_inputs( + (machines, flavors, products, profiles, testsuites, jobtemplates) = merge_inputs( args.files, validate=args.validate, clean=args.clean) jobtemplates.extend(generate_job_templates(products, profiles, testsuites)) - (machines, products, testsuites) = reverse_qol(machines, products, testsuites) + (machines, products, testsuites) = reverse_qol(machines, flavors, products, testsuites) # now produce the output in upstream-compatible format out = {} if jobtemplates: diff --git a/schemas/fif-complete.json b/schemas/fif-complete.json index 913b9577..b99735ba 100644 --- a/schemas/fif-complete.json +++ b/schemas/fif-complete.json @@ -11,6 +11,7 @@ ], "properties": { "Machines": { "$ref": "fif-machines.json" }, + "Flavors": { "$ref": "fif-flavors.json" }, "Products": { "$ref": "fif-products.json" }, "Profiles": { "$ref": "fif-profiles.json" }, "TestSuites": { "$ref": "fif-testsuites.json" }, diff --git a/schemas/fif-flavor.json b/schemas/fif-flavor.json new file mode 100644 index 00000000..e47fe4fc --- /dev/null +++ b/schemas/fif-flavor.json @@ -0,0 +1,11 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "fif-flavor.json", + "title": "FIF single flavor schema", + "type": "object", + "required": ["settings"], + "properties": { + "settings": { "$ref": "fif-settingshash.json" } + }, + "additionalProperties": false +} diff --git a/schemas/fif-flavors.json b/schemas/fif-flavors.json new file mode 100644 index 00000000..92fbb4d0 --- /dev/null +++ b/schemas/fif-flavors.json @@ -0,0 +1,8 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "fif-flavors.json", + "title": "FIF Flavors object schema", + "type": "object", + "minProperties": 1, + "additionalProperties": { "$ref": "fif-flavor.json" } +} diff --git a/schemas/fif-incomplete.json b/schemas/fif-incomplete.json index c5a6b473..ba212dd5 100644 --- a/schemas/fif-incomplete.json +++ b/schemas/fif-incomplete.json @@ -11,6 +11,7 @@ ], "properties": { "Machines": { "$ref": "fif-machines.json" }, + "Flavors": { "$ref": "fif-flavors.json" }, "Products": { "$ref": "fif-products.json" }, "Profiles": { "$ref": "fif-profiles.json" }, "TestSuites": { "$ref": "fif-testsuites.json" }, diff --git a/unittests/data/templates.fif.json b/unittests/data/templates.fif.json index 16040ac3..77fbb7cf 100644 --- a/unittests/data/templates.fif.json +++ b/unittests/data/templates.fif.json @@ -28,14 +28,19 @@ } } }, + "Flavors": { + "Server-dvd-iso": { + "settings": { + "TEST_TARGET": "ISO", + "RETRY": "1" + } + } + }, "Products": { "fedora-Server-dvd-iso-ppc64le-*": { "arch": "ppc64le", "distri": "fedora", "flavor": "Server-dvd-iso", - "settings": { - "TEST_TARGET": "ISO" - }, "version": "*" }, "fedora-Server-dvd-iso-x86_64-*": { @@ -43,7 +48,8 @@ "distri": "fedora", "flavor": "Server-dvd-iso", "settings": { - "TEST_TARGET": "ISO" + "TEST_TARGET": "COMPOSE", + "QEMURAM": "3072" }, "version": "*" } diff --git a/unittests/test_fifloader.py b/unittests/test_fifloader.py index 46971029..eeeb7339 100644 --- a/unittests/test_fifloader.py +++ b/unittests/test_fifloader.py @@ -71,9 +71,10 @@ def test_schema_validate(): ) def test_merge_inputs(input1, input2): """Test for merge_inputs.""" - (machines, products, profiles, testsuites, jobtemplates) = _get_merged(input1, input2) + (machines, flavors, products, profiles, testsuites, jobtemplates) = _get_merged(input1, input2) # a few known attributes of the test data to ensure the merge worked assert len(machines) == 2 + assert len(flavors) == 1 assert len(products) == 4 assert len(profiles) == 4 assert not jobtemplates @@ -90,7 +91,7 @@ def test_merge_inputs(input1, input2): def test_generate_job_templates(): """Test for generate_job_templates.""" - (machines, products, profiles, testsuites, _) = _get_merged() + (machines, _, products, profiles, testsuites, _) = _get_merged() templates = fifloader.generate_job_templates(products, profiles, testsuites) # we should get one template per profile in merged input assert len(templates) == 8 @@ -106,8 +107,8 @@ def test_generate_job_templates(): def test_reverse_qol(): """Test for reverse_qol.""" - (machines, products, _, testsuites, _) = _get_merged() - (machines, products, testsuites) = fifloader.reverse_qol(machines, products, testsuites) + (machines, flavors, products, _, testsuites, _) = _get_merged() + (machines, products, testsuites) = fifloader.reverse_qol(machines, flavors, products, testsuites) assert isinstance(machines, list) assert isinstance(products, list) assert isinstance(testsuites, list) @@ -125,6 +126,18 @@ def test_reverse_qol(): assert isinstance(settlist, list) for setting in settlist: assert list(setting.keys()) == ['key', 'value'] + # check flavor merge worked + sdixprod = [prod for prod in products if prod["name"] == "fedora-Server-dvd-iso-x86_64-*"][0] + sdipprod = [prod for prod in products if prod["name"] == "fedora-Server-dvd-iso-ppc64le-*"][0] + assert sdipprod["settings"] == [ + {"key": "TEST_TARGET", "value": "ISO"}, + {"key": "RETRY", "value": "1"} + ] + assert sdixprod["settings"] == [ + {"key": "TEST_TARGET", "value": "COMPOSE"}, + {"key": "RETRY", "value": "1"}, + {"key": "QEMURAM", "value": "3072"} + ] def test_parse_args(): """Test for parse_args."""