fifloader: add ProductDefaults

This adds another new fifloader-only top-level concept,
ProductDefaults. This just contains default values for all
products (on a per-file basis; these are applied *before* file
merge happens).

Signed-off-by: Adam Williamson <awilliam@redhat.com>

fifloader.py

@@ -26,7 +26,7 @@ can write this data to a JSON file and/or call the upstream loader on it directl
 the command-line arguments specified.
 
 The input data must contain definitions of Machines, Products, TestSuites, and Profiles. It may
-also contain Flavors. It also *may* contain JobTemplates, but does not have to and is expected to
+also contain Flavors and ProductDefaults. It also *may* contain JobTemplates, but is expected to
 contain none or only a few oddballs.
 
 The format for Machines, Products and TestSuites is based on the upstream format but with various
@@ -47,6 +47,11 @@ exists in the Flavors dict and defines any settings. If both the Product and the
 a given setting, the Product's definition wins. The purpose of the Flavors dict is to reduce
 duplication of settings between multiple products with the same flavor.
 
+The ProductDefaults dict contains default values for Products. Any key/value pair in this dict
+will be merged into every Product in the *same file*. Conflicts are resolved in favor of the
+Product, naturally. Note that this merge happens *before* the file merge, so ProductDefaults are
+*per file*, they are not merged from multiple input files as described below.
+
 The expected format of the Profiles dict is a dict-of-dicts. For each entry, the key is a unique
 name, and the value is a dict with keys 'machine' and 'product', each value being a valid name from
 the Machines or Products dict respectively. The name of each profile can be anything as long as
@@ -79,7 +84,7 @@ complete TestSuite definition, with the value of its `profiles` key as `{'foo':
 file may contain a TestSuite entry with the same key (name) as the complete definition in the other
 file, and the value as a dict with only a `profiles` key (with the value `{'bar': 20}`). This
 loader will combine those into a single complete TestSuite entry with the `profiles` value
-`{'foo': 10, 'bar': 20}`.
+`{'foo': 10, 'bar': 20}`. As noted above, ProductDefaults are *not* merged in this way.
 """
 
 import argparse
@@ -92,7 +97,7 @@ import jsonschema
 
 SCHEMAPATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'schemas')
 
-def schema_validate(instance, fif=True, complete=True, schemapath=SCHEMAPATH):
+def schema_validate(instance, fif=True, state='complete', schemapath=SCHEMAPATH):
     """Validate some input against one of our JSON schemas. We have
     'complete' and 'incomplete' schemas for FIF and the upstream
     template format. The 'complete' schemas expect the validated
@@ -106,10 +111,8 @@ def schema_validate(instance, fif=True, complete=True, schemapath=SCHEMAPATH):
     filename = 'openqa-'
     if fif:
         filename = 'fif-'
-    if complete:
-        filename += 'complete.json'
-    else:
-        filename += 'incomplete.json'
+    filename += state
+    filename += '.json'
     base_uri = "file://{0}/".format(schemapath)
     resolver = jsonschema.RefResolver(base_uri, None)
     schemafile = os.path.join(schemapath, filename)
@@ -126,8 +129,8 @@ def merge_inputs(inputs, validate=False, clean=False):
     """Merge multiple input files. Expects JSON file names. Optionally
     validates the input files before merging, and the merged output.
     Returns a 6-tuple of machines, flavors, products, profiles,
-    testsuites and jobtemplates (the first four as dicts, the fifth as
-    a list).
+    testsuites and jobtemplates (the first five as dicts, the last as a
+    list).
     """
     machines = {}
     flavors = {}
@@ -145,9 +148,16 @@ def merge_inputs(inputs, validate=False, clean=False):
         except Exception as err:
             print("Reading input file {} failed!".format(_input))
             sys.exit(str(err))
+        # validate against pre-products-merge schema
+        if validate:
+            schema_validate(data, fif=True, state="predefault")
+        for (pname, product) in data["Products"].items():
+            temp = dict(data.get("ProductDefaults", {}))
+            temp.update(product)
+            data["Products"][pname] = temp
         # validate against incomplete schema
         if validate:
-            schema_validate(data, fif=True, complete=False)
+            schema_validate(data, fif=True, state="incomplete")
 
         # simple merges for all these
         for (datatype, tgt) in (
@@ -195,7 +205,10 @@ def merge_inputs(inputs, validate=False, clean=False):
             merged['TestSuites'] = testsuites
         if jobtemplates:
             merged['JobTemplates'] = jobtemplates
-        schema_validate(merged, fif=True, complete=clean)
+        state = "incomplete"
+        if clean:
+            state = "complete"
+        schema_validate(merged, fif=True, state=state)
         print("Input template data is valid")
 
     return (machines, flavors, products, profiles, testsuites, jobtemplates)
@@ -346,7 +359,10 @@ def run(args):
         out['TestSuites'] = testsuites
     if args.validate:
         # validate generated data against upstream schema
-        schema_validate(out, fif=False, complete=args.clean)
+        state = "incomplete"
+        if args.clean:
+            state = "complete"
+        schema_validate(out, fif=False, state=state)
         print("Generated template data is valid")
     if args.write:
         # write generated output to given filename

schemas/fif-complete.json

@@ -12,6 +12,7 @@
   "properties": {
     "Machines": { "$ref": "fif-machines.json" },
     "Flavors": { "$ref": "fif-flavors.json" },
+    "ProductDefaults": { "$ref": "fif-productdefaults.json" },
     "Products": { "$ref": "fif-products.json" },
     "Profiles": { "$ref": "fif-profiles.json" },
     "TestSuites": { "$ref": "fif-testsuites.json" },

schemas/fif-incomplete.json

@@ -12,6 +12,7 @@
   "properties": {
     "Machines": { "$ref": "fif-machines.json" },
     "Flavors": { "$ref": "fif-flavors.json" },
+    "ProductDefaults": { "$ref": "fif-productdefaults.json" },
     "Products": { "$ref": "fif-products.json" },
     "Profiles": { "$ref": "fif-profiles.json" },
     "TestSuites": { "$ref": "fif-testsuites.json" },

schemas/fif-predefault.json

@@ -0,0 +1,23 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "fif-incomplete.json",
+  "type": "object",
+  "title": "Schema for Fedora Intermediate Format (FIF) openQA job template data before products defaults merge",
+  "anyOf": [
+    { "required": [ "Machines" ]},
+    { "required": [ "Products" ]},
+    { "required": [ "Profiles" ]},
+    { "required": [ "TestSuites" ]}
+  ],
+  "properties": {
+    "Machines": { "$ref": "fif-machines.json" },
+    "Flavors": { "$ref": "fif-flavors.json" },
+    "ProductDefaults": { "$ref": "fif-productdefaults.json" },
+    "Products": { "$ref": "fif-products-predefault.json" },
+    "Profiles": { "$ref": "fif-profiles.json" },
+    "TestSuites": { "$ref": "fif-testsuites.json" },
+    "JobTemplates": { "$ref": "openqa-jobtemplates.json" }
+  },
+  "additionalProperties": false
+}
+

schemas/fif-product-predefault.json

@@ -0,0 +1,15 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "fif-product.json",
+  "title": "FIF single product schema (pre-defaults-merge)",
+  "type": "object",
+  "properties": {
+    "arch": { "$ref": "fif-arch.json" },
+    "distri": { "$ref": "fif-distri.json" },
+    "flavor": { "type": "string" },
+    "version": { "$ref": "fif-version.json" },
+    "settings": { "$ref": "fif-settingshash.json" },
+    "name": { "type": "string" }
+  },
+  "additionalProperties": false
+}

schemas/fif-productdefaults.json

@@ -0,0 +1,17 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "fif-productdefaults.json",
+  "title": "FIF ProductDefaults object schema",
+  "type": "object",
+  "minProperties": 1,
+  "properties": {
+    "arch": { "$ref": "fif-arch.json" },
+    "distri": { "$ref": "fif-distri.json" },
+    "flavor": { "type": "string" },
+    "version": { "$ref": "fif-version.json" },
+    "settings": { "$ref": "fif-settingshash.json" },
+    "name": { "type": "string" }
+  },
+  "additionalProperties": false
+}
+

schemas/fif-products-predefault.json

@@ -0,0 +1,8 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "fif-products.json",
+  "title": "FIF Products object schema (pre-defaults-merge)",
+  "type": "object",
+  "minProperties": 1,
+  "additionalProperties": { "$ref": "fif-product-predefault.json" }
+}

unittests/data/templates.complete.fif.json

@@ -0,0 +1,86 @@
+{
+    "Machines": {
+        "64bit": {
+            "backend": "qemu",
+            "settings": {
+                "ARCH_BASE_MACHINE": "64bit",
+                "PART_TABLE_TYPE": "mbr",
+                "QEMUCPU": "Nehalem",
+                "QEMUCPUS": "2",
+                "QEMURAM": "2048",
+                "QEMUVGA": "virtio",
+                "QEMU_VIRTIO_RNG": "1",
+                "WORKER_CLASS": "qemu_x86_64"
+            }
+        },
+        "ppc64le": {
+            "backend": "qemu",
+            "settings": {
+                "ARCH_BASE_MACHINE": "ppc64le",
+                "OFW": 1,
+                "PART_TABLE_TYPE": "mbr",
+                "QEMU": "ppc64",
+                "QEMUCPU": "host",
+                "QEMURAM": 4096,
+                "QEMUVGA": "virtio",
+                "QEMU_VIRTIO_RNG": "1",
+                "WORKER_CLASS": "qemu_ppc64le"
+            }
+        }
+    },
+    "Products": {
+        "fedora-Server-dvd-iso-ppc64le-*": {
+            "distri": "fedora",
+            "arch": "ppc64le",
+            "flavor": "Server-dvd-iso",
+            "version": "*"
+        },
+        "fedora-Server-dvd-iso-x86_64-*": {
+            "distri": "fedora",
+            "arch": "x86_64",
+            "flavor": "Server-dvd-iso",
+            "settings": {
+                "TEST_TARGET": "COMPOSE",
+                "QEMURAM": "3072"
+            },
+            "version": "*"
+        }
+    },
+    "Profiles": {
+        "fedora-Server-dvd-iso-ppc64le-*-ppc64le": {
+            "machine": "ppc64le",
+            "product": "fedora-Server-dvd-iso-ppc64le-*"
+        },
+        "fedora-Server-dvd-iso-x86_64-*-64bit": {
+            "machine": "64bit",
+            "product": "fedora-Server-dvd-iso-x86_64-*"
+        }
+    },
+    "TestSuites": {
+        "base_selinux": {
+            "profiles": {
+                "fedora-Server-dvd-iso-ppc64le-*-ppc64le": 40,
+                "fedora-Server-dvd-iso-x86_64-*-64bit": 40
+            },
+            "settings": {
+                "BOOTFROM": "c",
+                "HDD_1": "disk_%FLAVOR%_%MACHINE%.qcow2",
+                "POSTINSTALL": "base_selinux",
+                "ROOT_PASSWORD": "weakpassword",
+                "START_AFTER_TEST": "install_default_upload",
+                "USER_LOGIN": "false"
+            }
+        },
+        "install_default_upload": {
+            "profiles": {
+                "fedora-Server-dvd-iso-ppc64le-*-ppc64le": 10,
+                "fedora-Server-dvd-iso-x86_64-*-64bit": 10
+            },
+            "settings": {
+                "PACKAGE_SET": "default",
+                "POSTINSTALL": "_collect_data",
+                "STORE_HDD_1": "disk_%FLAVOR%_%MACHINE%.qcow2"
+            }
+        }
+    }
+}

unittests/data/templates.fif.json

@@ -36,22 +36,23 @@
             }
         }
     },
+    "ProductDefaults": {
+        "distri": "fedora",
+        "version": "*"
+    },
     "Products": {
         "fedora-Server-dvd-iso-ppc64le-*": {
             "arch": "ppc64le",
-            "distri": "fedora",
-            "flavor": "Server-dvd-iso",
-            "version": "*"
+            "flavor": "Server-dvd-iso"
         },
         "fedora-Server-dvd-iso-x86_64-*": {
             "arch": "x86_64",
-            "distri": "fedora",
             "flavor": "Server-dvd-iso",
             "settings": {
                 "TEST_TARGET": "COMPOSE",
                 "QEMURAM": "3072"
             },
-            "version": "*"
+            "version": "Rawhide"
         }
     },
     "Profiles": {

unittests/test_fifloader.py

@@ -45,19 +45,21 @@ def _get_merged(input1='templates.fif.json', input2='templates-updates.fif.json'
 
 def test_schema_validate():
     """Test for schema_validate."""
-    with open(os.path.join(DATAPATH, 'templates.fif.json'), 'r') as tempfh:
+    # this one has no Flavors and complete Products, to check such a
+    # layout matches the 'complete' schema as it should
+    with open(os.path.join(DATAPATH, 'templates.complete.fif.json'), 'r') as tempfh:
         tempdata = json.load(tempfh)
     with open(os.path.join(DATAPATH, 'templates-updates.fif.json'), 'r') as updfh:
         updata = json.load(updfh)
-    assert fifloader.schema_validate(tempdata, fif=True, complete=True) is True
-    assert fifloader.schema_validate(tempdata, fif=True, complete=False) is True
-    assert fifloader.schema_validate(updata, fif=True, complete=False) is True
+    assert fifloader.schema_validate(tempdata, fif=True, state="complete") is True
+    assert fifloader.schema_validate(tempdata, fif=True, state="incomplete") is True
+    assert fifloader.schema_validate(updata, fif=True, state="incomplete") is True
     with pytest.raises(jsonschema.exceptions.ValidationError):
-        fifloader.schema_validate(updata, fif=True, complete=True)
+        fifloader.schema_validate(updata, fif=True, state="complete")
     with pytest.raises(jsonschema.exceptions.ValidationError):
-        fifloader.schema_validate(tempdata, fif=False, complete=True)
+        fifloader.schema_validate(tempdata, fif=False, state="complete")
     with pytest.raises(jsonschema.exceptions.ValidationError):
-        fifloader.schema_validate(tempdata, fif=False, complete=False)
+        fifloader.schema_validate(tempdata, fif=False, state="incomplete")
     # we test successful openQA validation later in test_run
 
 # we test merging in both orders, because it can work in one order
@@ -88,6 +90,11 @@ def test_merge_inputs(input1, input2):
     # and we should still have the settings (note, combining settings
     # is not supported, the last-read settings dict is always used)
     assert len(testsuites['base_selinux']['settings']) == 6
+    # check product defaults were merged correctly
+    assert products['fedora-Server-dvd-iso-ppc64le-*']['distri'] == 'fedora'
+    assert products['fedora-Server-dvd-iso-ppc64le-*']['version'] == '*'
+    assert products['fedora-Server-dvd-iso-x86_64-*']['distri'] == 'fedora'
+    assert products['fedora-Server-dvd-iso-x86_64-*']['version'] == 'Rawhide'
 
 def test_generate_job_templates():
     """Test for generate_job_templates."""
@@ -184,7 +191,7 @@ def test_run(fakerun):
                        os.path.join(DATAPATH, 'templates-updates.fif.json')])
         written = json.load(tempfh)
     # check written data matches upstream data schema
-    assert fifloader.schema_validate(written, fif=False, complete=True) is True
+    assert fifloader.schema_validate(written, fif=False, state="complete") is True
     # test the loader stuff, first with one failure of subprocess.run
     # and success on the second try:
     fakerun.side_effect=[subprocess.CalledProcessError(1, "cmd"), True]