os-autoinst-distri-fedora/fifconverter

#!/bin/python3

"""
This script takes JSON-formatted openQA template data (in the older format with a JobTemplates
dict, not the newer YAML-ish format organized by job group) and converts to an intermediate format
(Fedora Intermediate Format - 'fif') intended to be easier for human editing. It extracts all the
unique 'environment profiles' - a combination of machine and product - from the JobTemplates and
stores them in a 'Profiles' dict; it then adds a 'profiles' key to each test suite, indicating
which profiles that suite is run on. It is fairly easy to reverse this process to reproduce the
openQA loader-compatible data, but the intermediate format is more friendly to a human editor.
Adding a new test suite to run on existing 'profiles' only requires adding the suite and an
appropriate 'profiles' dict. Adding a new profile involves adding the machine and/or product,
manually adding the profile to the Profiles dict, and then adding the profile to all the test
suites which should be run on it. See also fifloader.py, which handles converting FIF input to
upstream format, and optionally can pass it through to the upstream loader.
"""

import json

with open('templates.old.json', 'r') as tempfh:
    tempdata = json.load(tempfh)
with open('templates-updates.old.json', 'r') as updfh:
    updata = json.load(updfh)

def _synthesize_product_name(product):
    """Synthesize a product name from a product dict. We do this when
    reading the templates file and also when constructing the profiles
    so use a function to make sure they both do it the same way.
    """
    return "-".join((product['distri'], product['flavor'], product['arch'], product['version']))

def read_templates(templates):
    newtemps = {}
    if 'Machines' in templates:
        newtemps['Machines'] = {}
        for machine in templates['Machines']:
            # condense the stupid settings format
            machine['settings'] = {settdict['key']: settdict['value'] for settdict in machine['settings']}
            # just use a dict, not a list of dicts with 'name' keys...
            name = machine.pop('name')
            newtemps['Machines'][name] = machine
    if 'Products' in templates:
        newtemps['Products'] = {}
        for product in templates['Products']:
            # condense the stupid settings format
            product['settings'] = {settdict['key']: settdict['value'] for settdict in product['settings']}
            # synthesize a name, as we don't have any in our templates
            # and we can use them in the scenarios. however, note that
            # openQA itself doesn't let you use the product name as a
            # key when loading templates, unlike the machine name, our
            # loader has to reverse this and provide the full product
            # dict to the upstream loader
            name = _synthesize_product_name(product)
            # this is always an empty string in our templates
            del product['name']
            newtemps['Products'][name] = product
    if 'TestSuites' in templates:
        newtemps['TestSuites'] = {}
        for testsuite in templates['TestSuites']:
            # condense the stupid settings format
            testsuite['settings'] = {settdict['key']: settdict['value'] for settdict in testsuite['settings']}
            # just use a dict, not a list of dicts with 'name' keys...
            name = testsuite.pop('name')
            newtemps['TestSuites'][name] = testsuite
    profiles = {}
    for jobtemp in templates['JobTemplates']:
        # figure out the profile for each job template and add it to
        # the dict. For Fedora, the group name is predictable based on
        # the arch and whether it's an update test; the intermediate
        # loader figures that out
        profile = {
            'machine': jobtemp['machine']['name'],
            'product': _synthesize_product_name(jobtemp['product']),
        }
        profname = '-'.join([profile['product'], profile['machine']])
        # keep track of all the profiles we've hit
        profiles[profname] = profile

        test = jobtemp['test_suite']['name']
        prio = jobtemp['prio']
        try:
            suite = newtemps['TestSuites'][test]
        except KeyError:
            # this is a templates-updates JobTemplate which refers to a
            # TestSuite defined in templates. What we do here is define
            # a partial TestSuite which contains only the name and the
            # profiles; the loader for this format knows how to combine
            # dicts (including incomplete ones) from multiple source
            # files into one big final-format lump
            suite = {}
            newtemps['TestSuites'][test] = suite
        if 'profiles' in suite:
            suite['profiles'][profname] = prio
        else:
            suite['profiles'] = {profname: prio}

    newtemps['Profiles'] = profiles
    return newtemps

with open('templates.fif.json', 'w') as newtempfh:
    json.dump(read_templates(tempdata), newtempfh, sort_keys=True, indent=4)

with open('templates-updates.fif.json', 'w') as newtempfh:
    json.dump(read_templates(updata), newtempfh, sort_keys=True, indent=4)