From ef33d00f5baf78566e4e8e324067a1fbe806a988 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Mon, 3 Feb 2020 12:43:27 +0100 Subject: [PATCH] Add --schema-override to pungi-config-validate script Some composes might need extra validation to ensure they are following certain strict rules - for example containing only signed packages or packages only from particular Koji tag. There is currently no way how to check that Pungi configuration fulfills these extra requirements. This commit adds new `--schema-override` option to `pungi-config-validate` script which allows caller to specify path to JSON schema overriding the default JSON schema and therefore limitting it further. For exmaple, to limit the `pkgset_source` to `koji`, one can use following JSON schema override: ``` { "properties": { "pkgset_source": { "enum": ["koji"] } } } ``` It is possible to use `--schema-override` multiple times to apply multiple schema overrides. Merges: https://pagure.io/pungi/pull-request/1341 Signed-off-by: Jan Kaluza --- pungi/checks.py | 23 +++++++++++++++++++++-- pungi/scripts/config_validate.py | 22 +++++++++++++++++++--- tests/data/dummy-override.json | 7 +++++++ tests/test_checks.py | 25 +++++++++++++++++++++++++ tests/test_config_validate_script.py | 14 ++++++++++++++ 5 files changed, 86 insertions(+), 5 deletions(-) create mode 100644 tests/data/dummy-override.json diff --git a/pungi/checks.py b/pungi/checks.py index 7eb98a2a..c67c88b9 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -185,12 +185,12 @@ def _check_dep(name, value, lst, matcher, fmt): yield fmt.format(name, value, dep) -def validate(config, offline=False): +def validate(config, offline=False, schema=None): """Test the configuration against schema. Undefined values for which a default value exists will be filled in. """ - schema = make_schema() + schema = schema or make_schema() DefaultValidator = _extend_with_default_and_alias( jsonschema.Draft4Validator, offline=offline ) @@ -1442,6 +1442,25 @@ CONFIG_DEPS = { } +def update_schema(schema, update_dict): + """ + Updates the leaf values in `schema` by the leaf values from `update_dict`. + + This is different from `schema.update(update_dict)`, because + the `schema.update()` would override also non-leaf values. + + :param schema: Schema to update. + :param updated_dict: Dict matching the schema with updated leaf values. + :returns: Updated schema + """ + for k, v in update_dict.items(): + if isinstance(v, dict): + schema[k] = update_schema(schema.get(k, {}), v) + else: + schema[k] = v + return schema + + def _get_gather_backends(): if six.PY2: return ['yum', 'dnf'] diff --git a/pungi/scripts/config_validate.py b/pungi/scripts/config_validate.py index 0fa57689..c73e3b91 100644 --- a/pungi/scripts/config_validate.py +++ b/pungi/scripts/config_validate.py @@ -72,7 +72,7 @@ def read_variants(compose, config): compose.all_variants[child.uid] = child -def run(config, topdir, has_old, offline, defined_variables): +def run(config, topdir, has_old, offline, defined_variables, schema_overrides): # Load default values for undefined variables. This is useful for # validating templates that are supposed to be filled in later with # pungi-config-dump. @@ -89,7 +89,13 @@ def run(config, topdir, has_old, offline, defined_variables): # Remove the dummy variables used for defaults. config_utils.remove_unknown(conf, defined_variables) - errors, warnings = pungi.checks.validate(conf, offline=offline) + # Load extra schemas JSON files. + schema = pungi.checks.make_schema() + for schema_override in schema_overrides: + with open(schema_override) as f: + schema = pungi.checks.update_schema(schema, json.load(f)) + + errors, warnings = pungi.checks.validate(conf, offline=offline, schema=schema) if errors or warnings: for error in errors + warnings: print(error) @@ -169,11 +175,21 @@ def main(args=None): "Can be used multiple times." ), ) + parser.add_argument( + '--schema-override', + action="append", + default=[], + help=( + 'Path to extra JSON schema defining the values which will override ' + 'the original Pungi JSON schema values.' + ), + ) opts = parser.parse_args(args) defines = config_utils.extract_defines(opts.define) with pungi.util.temp_dir() as topdir: - errors = run(opts.config, topdir, opts.old_composes, opts.offline, defines) + errors = run(opts.config, topdir, opts.old_composes, opts.offline, defines, + opts.schema_override) for msg in errors: print(msg) diff --git a/tests/data/dummy-override.json b/tests/data/dummy-override.json new file mode 100644 index 00000000..39189f62 --- /dev/null +++ b/tests/data/dummy-override.json @@ -0,0 +1,7 @@ +{ + "properties": { + "pkgset_source": { + "enum": ["koji"] + } + } +} diff --git a/tests/test_checks.py b/tests/test_checks.py index 299bca80..4b945a0d 100644 --- a/tests/test_checks.py +++ b/tests/test_checks.py @@ -542,6 +542,31 @@ class TestSchemaValidator(unittest.TestCase): self.assertEqual(config["foo"], "git://example.com/repo.git#HEAD") self.assertEqual(resolve_git_url.call_args_list, []) + def test_update_schema(self): + schema = checks.make_schema() + schema_override = { + "definitions": { + "scm_dict": { + "properties": { + "scm": { + "enum": ["git"] + }, + "repo": { + "enum": ["git://localhost/pungi-fedora.git"] + }, + } + } + } + } + schema = checks.update_schema(schema, schema_override) + scm_dict_properties = schema["definitions"]["scm_dict"]["properties"] + self.assertEqual( + scm_dict_properties["scm"], + {'enum': ['git'], 'type': 'string'}) + self.assertEqual( + scm_dict_properties["repo"], + {'enum': ['git://localhost/pungi-fedora.git'], 'type': 'string'}) + self.assertEqual(scm_dict_properties["file"], {"type": "string"}) class TestUmask(unittest.TestCase): def setUp(self): diff --git a/tests/test_config_validate_script.py b/tests/test_config_validate_script.py index 961e8701..542f35b3 100644 --- a/tests/test_config_validate_script.py +++ b/tests/test_config_validate_script.py @@ -13,6 +13,7 @@ from tests import helpers HERE = os.path.abspath(os.path.dirname(__file__)) DUMMY_CONFIG = os.path.join(HERE, 'data/dummy-pungi.conf') +SCHEMA_OVERRIDE = os.path.join(HERE, 'data/dummy-override.json') class ConfigValidateScriptTest(helpers.PungiTestCase): @@ -24,3 +25,16 @@ class ConfigValidateScriptTest(helpers.PungiTestCase): cli_main() self.assertEqual('', stdout.getvalue()) self.assertEqual('', stderr.getvalue()) + + @mock.patch('sys.argv', new=[ + 'pungi-config-validate', DUMMY_CONFIG, "--schema-override", + SCHEMA_OVERRIDE]) + @mock.patch('sys.stderr', new_callable=six.StringIO) + @mock.patch('sys.stdout', new_callable=six.StringIO) + @mock.patch('sys.exit') + def test_schema_override(self, exit, stdout, stderr): + cli_main() + self.assertTrue(stdout.getvalue().startswith( + "Failed validation in pkgset_source: 'repos' is not one of")) + self.assertEqual('', stderr.getvalue()) + exit.assert_called_once_with(1)