From 99638a0c25f57ae7214e7c836674d280a73e1a5c Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Sat, 10 May 2025 12:30:07 -0700 Subject: [PATCH] fifloader: add ProductDefaults This adds another new fifloader-only top-level concept, ProductDefaults. This just contains default values for all products (on a per-file basis; these are applied *before* file merge happens). Signed-off-by: Adam Williamson --- fifloader.py | 40 +++++++--- schemas/fif-complete.json | 1 + schemas/fif-incomplete.json | 1 + schemas/fif-predefault.json | 23 ++++++ schemas/fif-product-predefault.json | 15 ++++ schemas/fif-productdefaults.json | 17 +++++ schemas/fif-products-predefault.json | 8 ++ unittests/data/templates.complete.fif.json | 86 ++++++++++++++++++++++ unittests/data/templates.fif.json | 11 +-- unittests/test_fifloader.py | 23 ++++-- 10 files changed, 200 insertions(+), 25 deletions(-) create mode 100644 schemas/fif-predefault.json create mode 100644 schemas/fif-product-predefault.json create mode 100644 schemas/fif-productdefaults.json create mode 100644 schemas/fif-products-predefault.json create mode 100644 unittests/data/templates.complete.fif.json diff --git a/fifloader.py b/fifloader.py index 0ead63d8..8c6e8947 100755 --- a/fifloader.py +++ b/fifloader.py @@ -26,7 +26,7 @@ can write this data to a JSON file and/or call the upstream loader on it directl the command-line arguments specified. The input data must contain definitions of Machines, Products, TestSuites, and Profiles. It may -also contain Flavors. It also *may* contain JobTemplates, but does not have to and is expected to +also contain Flavors and ProductDefaults. It also *may* contain JobTemplates, but is expected to contain none or only a few oddballs. The format for Machines, Products and TestSuites is based on the upstream format but with various @@ -47,6 +47,11 @@ exists in the Flavors dict and defines any settings. If both the Product and the a given setting, the Product's definition wins. The purpose of the Flavors dict is to reduce duplication of settings between multiple products with the same flavor. +The ProductDefaults dict contains default values for Products. Any key/value pair in this dict +will be merged into every Product in the *same file*. Conflicts are resolved in favor of the +Product, naturally. Note that this merge happens *before* the file merge, so ProductDefaults are +*per file*, they are not merged from multiple input files as described below. + The expected format of the Profiles dict is a dict-of-dicts. For each entry, the key is a unique name, and the value is a dict with keys 'machine' and 'product', each value being a valid name from the Machines or Products dict respectively. The name of each profile can be anything as long as @@ -79,7 +84,7 @@ complete TestSuite definition, with the value of its `profiles` key as `{'foo': file may contain a TestSuite entry with the same key (name) as the complete definition in the other file, and the value as a dict with only a `profiles` key (with the value `{'bar': 20}`). This loader will combine those into a single complete TestSuite entry with the `profiles` value -`{'foo': 10, 'bar': 20}`. +`{'foo': 10, 'bar': 20}`. As noted above, ProductDefaults are *not* merged in this way. """ import argparse @@ -92,7 +97,7 @@ import jsonschema SCHEMAPATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'schemas') -def schema_validate(instance, fif=True, complete=True, schemapath=SCHEMAPATH): +def schema_validate(instance, fif=True, state='complete', schemapath=SCHEMAPATH): """Validate some input against one of our JSON schemas. We have 'complete' and 'incomplete' schemas for FIF and the upstream template format. The 'complete' schemas expect the validated @@ -106,10 +111,8 @@ def schema_validate(instance, fif=True, complete=True, schemapath=SCHEMAPATH): filename = 'openqa-' if fif: filename = 'fif-' - if complete: - filename += 'complete.json' - else: - filename += 'incomplete.json' + filename += state + filename += '.json' base_uri = "file://{0}/".format(schemapath) resolver = jsonschema.RefResolver(base_uri, None) schemafile = os.path.join(schemapath, filename) @@ -126,8 +129,8 @@ def merge_inputs(inputs, validate=False, clean=False): """Merge multiple input files. Expects JSON file names. Optionally validates the input files before merging, and the merged output. Returns a 6-tuple of machines, flavors, products, profiles, - testsuites and jobtemplates (the first four as dicts, the fifth as - a list). + testsuites and jobtemplates (the first five as dicts, the last as a + list). """ machines = {} flavors = {} @@ -145,9 +148,16 @@ def merge_inputs(inputs, validate=False, clean=False): except Exception as err: print("Reading input file {} failed!".format(_input)) sys.exit(str(err)) + # validate against pre-products-merge schema + if validate: + schema_validate(data, fif=True, state="predefault") + for (pname, product) in data["Products"].items(): + temp = dict(data.get("ProductDefaults", {})) + temp.update(product) + data["Products"][pname] = temp # validate against incomplete schema if validate: - schema_validate(data, fif=True, complete=False) + schema_validate(data, fif=True, state="incomplete") # simple merges for all these for (datatype, tgt) in ( @@ -195,7 +205,10 @@ def merge_inputs(inputs, validate=False, clean=False): merged['TestSuites'] = testsuites if jobtemplates: merged['JobTemplates'] = jobtemplates - schema_validate(merged, fif=True, complete=clean) + state = "incomplete" + if clean: + state = "complete" + schema_validate(merged, fif=True, state=state) print("Input template data is valid") return (machines, flavors, products, profiles, testsuites, jobtemplates) @@ -346,7 +359,10 @@ def run(args): out['TestSuites'] = testsuites if args.validate: # validate generated data against upstream schema - schema_validate(out, fif=False, complete=args.clean) + state = "incomplete" + if args.clean: + state = "complete" + schema_validate(out, fif=False, state=state) print("Generated template data is valid") if args.write: # write generated output to given filename diff --git a/schemas/fif-complete.json b/schemas/fif-complete.json index b99735ba..75731871 100644 --- a/schemas/fif-complete.json +++ b/schemas/fif-complete.json @@ -12,6 +12,7 @@ "properties": { "Machines": { "$ref": "fif-machines.json" }, "Flavors": { "$ref": "fif-flavors.json" }, + "ProductDefaults": { "$ref": "fif-productdefaults.json" }, "Products": { "$ref": "fif-products.json" }, "Profiles": { "$ref": "fif-profiles.json" }, "TestSuites": { "$ref": "fif-testsuites.json" }, diff --git a/schemas/fif-incomplete.json b/schemas/fif-incomplete.json index ba212dd5..68dd2a81 100644 --- a/schemas/fif-incomplete.json +++ b/schemas/fif-incomplete.json @@ -12,6 +12,7 @@ "properties": { "Machines": { "$ref": "fif-machines.json" }, "Flavors": { "$ref": "fif-flavors.json" }, + "ProductDefaults": { "$ref": "fif-productdefaults.json" }, "Products": { "$ref": "fif-products.json" }, "Profiles": { "$ref": "fif-profiles.json" }, "TestSuites": { "$ref": "fif-testsuites.json" }, diff --git a/schemas/fif-predefault.json b/schemas/fif-predefault.json new file mode 100644 index 00000000..f8ea7f60 --- /dev/null +++ b/schemas/fif-predefault.json @@ -0,0 +1,23 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "fif-incomplete.json", + "type": "object", + "title": "Schema for Fedora Intermediate Format (FIF) openQA job template data before products defaults merge", + "anyOf": [ + { "required": [ "Machines" ]}, + { "required": [ "Products" ]}, + { "required": [ "Profiles" ]}, + { "required": [ "TestSuites" ]} + ], + "properties": { + "Machines": { "$ref": "fif-machines.json" }, + "Flavors": { "$ref": "fif-flavors.json" }, + "ProductDefaults": { "$ref": "fif-productdefaults.json" }, + "Products": { "$ref": "fif-products-predefault.json" }, + "Profiles": { "$ref": "fif-profiles.json" }, + "TestSuites": { "$ref": "fif-testsuites.json" }, + "JobTemplates": { "$ref": "openqa-jobtemplates.json" } + }, + "additionalProperties": false +} + diff --git a/schemas/fif-product-predefault.json b/schemas/fif-product-predefault.json new file mode 100644 index 00000000..00b2a7be --- /dev/null +++ b/schemas/fif-product-predefault.json @@ -0,0 +1,15 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "fif-product.json", + "title": "FIF single product schema (pre-defaults-merge)", + "type": "object", + "properties": { + "arch": { "$ref": "fif-arch.json" }, + "distri": { "$ref": "fif-distri.json" }, + "flavor": { "type": "string" }, + "version": { "$ref": "fif-version.json" }, + "settings": { "$ref": "fif-settingshash.json" }, + "name": { "type": "string" } + }, + "additionalProperties": false +} diff --git a/schemas/fif-productdefaults.json b/schemas/fif-productdefaults.json new file mode 100644 index 00000000..e1c20714 --- /dev/null +++ b/schemas/fif-productdefaults.json @@ -0,0 +1,17 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "fif-productdefaults.json", + "title": "FIF ProductDefaults object schema", + "type": "object", + "minProperties": 1, + "properties": { + "arch": { "$ref": "fif-arch.json" }, + "distri": { "$ref": "fif-distri.json" }, + "flavor": { "type": "string" }, + "version": { "$ref": "fif-version.json" }, + "settings": { "$ref": "fif-settingshash.json" }, + "name": { "type": "string" } + }, + "additionalProperties": false +} + diff --git a/schemas/fif-products-predefault.json b/schemas/fif-products-predefault.json new file mode 100644 index 00000000..e11a806d --- /dev/null +++ b/schemas/fif-products-predefault.json @@ -0,0 +1,8 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "fif-products.json", + "title": "FIF Products object schema (pre-defaults-merge)", + "type": "object", + "minProperties": 1, + "additionalProperties": { "$ref": "fif-product-predefault.json" } +} diff --git a/unittests/data/templates.complete.fif.json b/unittests/data/templates.complete.fif.json new file mode 100644 index 00000000..4a056ec6 --- /dev/null +++ b/unittests/data/templates.complete.fif.json @@ -0,0 +1,86 @@ +{ + "Machines": { + "64bit": { + "backend": "qemu", + "settings": { + "ARCH_BASE_MACHINE": "64bit", + "PART_TABLE_TYPE": "mbr", + "QEMUCPU": "Nehalem", + "QEMUCPUS": "2", + "QEMURAM": "2048", + "QEMUVGA": "virtio", + "QEMU_VIRTIO_RNG": "1", + "WORKER_CLASS": "qemu_x86_64" + } + }, + "ppc64le": { + "backend": "qemu", + "settings": { + "ARCH_BASE_MACHINE": "ppc64le", + "OFW": 1, + "PART_TABLE_TYPE": "mbr", + "QEMU": "ppc64", + "QEMUCPU": "host", + "QEMURAM": 4096, + "QEMUVGA": "virtio", + "QEMU_VIRTIO_RNG": "1", + "WORKER_CLASS": "qemu_ppc64le" + } + } + }, + "Products": { + "fedora-Server-dvd-iso-ppc64le-*": { + "distri": "fedora", + "arch": "ppc64le", + "flavor": "Server-dvd-iso", + "version": "*" + }, + "fedora-Server-dvd-iso-x86_64-*": { + "distri": "fedora", + "arch": "x86_64", + "flavor": "Server-dvd-iso", + "settings": { + "TEST_TARGET": "COMPOSE", + "QEMURAM": "3072" + }, + "version": "*" + } + }, + "Profiles": { + "fedora-Server-dvd-iso-ppc64le-*-ppc64le": { + "machine": "ppc64le", + "product": "fedora-Server-dvd-iso-ppc64le-*" + }, + "fedora-Server-dvd-iso-x86_64-*-64bit": { + "machine": "64bit", + "product": "fedora-Server-dvd-iso-x86_64-*" + } + }, + "TestSuites": { + "base_selinux": { + "profiles": { + "fedora-Server-dvd-iso-ppc64le-*-ppc64le": 40, + "fedora-Server-dvd-iso-x86_64-*-64bit": 40 + }, + "settings": { + "BOOTFROM": "c", + "HDD_1": "disk_%FLAVOR%_%MACHINE%.qcow2", + "POSTINSTALL": "base_selinux", + "ROOT_PASSWORD": "weakpassword", + "START_AFTER_TEST": "install_default_upload", + "USER_LOGIN": "false" + } + }, + "install_default_upload": { + "profiles": { + "fedora-Server-dvd-iso-ppc64le-*-ppc64le": 10, + "fedora-Server-dvd-iso-x86_64-*-64bit": 10 + }, + "settings": { + "PACKAGE_SET": "default", + "POSTINSTALL": "_collect_data", + "STORE_HDD_1": "disk_%FLAVOR%_%MACHINE%.qcow2" + } + } + } +} diff --git a/unittests/data/templates.fif.json b/unittests/data/templates.fif.json index 77fbb7cf..62e384c9 100644 --- a/unittests/data/templates.fif.json +++ b/unittests/data/templates.fif.json @@ -36,22 +36,23 @@ } } }, + "ProductDefaults": { + "distri": "fedora", + "version": "*" + }, "Products": { "fedora-Server-dvd-iso-ppc64le-*": { "arch": "ppc64le", - "distri": "fedora", - "flavor": "Server-dvd-iso", - "version": "*" + "flavor": "Server-dvd-iso" }, "fedora-Server-dvd-iso-x86_64-*": { "arch": "x86_64", - "distri": "fedora", "flavor": "Server-dvd-iso", "settings": { "TEST_TARGET": "COMPOSE", "QEMURAM": "3072" }, - "version": "*" + "version": "Rawhide" } }, "Profiles": { diff --git a/unittests/test_fifloader.py b/unittests/test_fifloader.py index eeeb7339..5a0b9128 100644 --- a/unittests/test_fifloader.py +++ b/unittests/test_fifloader.py @@ -45,19 +45,21 @@ def _get_merged(input1='templates.fif.json', input2='templates-updates.fif.json' def test_schema_validate(): """Test for schema_validate.""" - with open(os.path.join(DATAPATH, 'templates.fif.json'), 'r') as tempfh: + # this one has no Flavors and complete Products, to check such a + # layout matches the 'complete' schema as it should + with open(os.path.join(DATAPATH, 'templates.complete.fif.json'), 'r') as tempfh: tempdata = json.load(tempfh) with open(os.path.join(DATAPATH, 'templates-updates.fif.json'), 'r') as updfh: updata = json.load(updfh) - assert fifloader.schema_validate(tempdata, fif=True, complete=True) is True - assert fifloader.schema_validate(tempdata, fif=True, complete=False) is True - assert fifloader.schema_validate(updata, fif=True, complete=False) is True + assert fifloader.schema_validate(tempdata, fif=True, state="complete") is True + assert fifloader.schema_validate(tempdata, fif=True, state="incomplete") is True + assert fifloader.schema_validate(updata, fif=True, state="incomplete") is True with pytest.raises(jsonschema.exceptions.ValidationError): - fifloader.schema_validate(updata, fif=True, complete=True) + fifloader.schema_validate(updata, fif=True, state="complete") with pytest.raises(jsonschema.exceptions.ValidationError): - fifloader.schema_validate(tempdata, fif=False, complete=True) + fifloader.schema_validate(tempdata, fif=False, state="complete") with pytest.raises(jsonschema.exceptions.ValidationError): - fifloader.schema_validate(tempdata, fif=False, complete=False) + fifloader.schema_validate(tempdata, fif=False, state="incomplete") # we test successful openQA validation later in test_run # we test merging in both orders, because it can work in one order @@ -88,6 +90,11 @@ def test_merge_inputs(input1, input2): # and we should still have the settings (note, combining settings # is not supported, the last-read settings dict is always used) assert len(testsuites['base_selinux']['settings']) == 6 + # check product defaults were merged correctly + assert products['fedora-Server-dvd-iso-ppc64le-*']['distri'] == 'fedora' + assert products['fedora-Server-dvd-iso-ppc64le-*']['version'] == '*' + assert products['fedora-Server-dvd-iso-x86_64-*']['distri'] == 'fedora' + assert products['fedora-Server-dvd-iso-x86_64-*']['version'] == 'Rawhide' def test_generate_job_templates(): """Test for generate_job_templates.""" @@ -184,7 +191,7 @@ def test_run(fakerun): os.path.join(DATAPATH, 'templates-updates.fif.json')]) written = json.load(tempfh) # check written data matches upstream data schema - assert fifloader.schema_validate(written, fif=False, complete=True) is True + assert fifloader.schema_validate(written, fif=False, state="complete") is True # test the loader stuff, first with one failure of subprocess.run # and success on the second try: fakerun.side_effect=[subprocess.CalledProcessError(1, "cmd"), True]