config-validate: Allow defining variables

When trying to validate a template that should later be filled in with
`pungi-config-dump`, there will be errors about undefined variables.
These are meant to be set when the template is populated.

This patch adds support for `-e`, `--define` argument to the validation
script that can be used to suppress these errors.

Alternatively a JSON file is read from the directory with config file
that can contain values for the variables.

The `--define` option is changed in both validation and dumping to allow
empty string as an accepted value.

JIRA: COMPOSE-3599
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

bin/pungi-config-dump

@@ -6,7 +6,6 @@ from __future__ import print_function
 import argparse
 import json
 import os
-import re
 import shutil
 import sys
 
@@ -16,6 +15,8 @@ import kobo.conf
 import pungi.checks
 import pungi.util
 
+from pungi_utils import config_utils
+
 
 def load_file(source, conf):
     try:
@@ -32,17 +33,6 @@ def load_source(source, conf):
         load_file(os.path.join(source, "logs/global/config-dump.global.log"), conf)
 
 
-def validate_definition(value):
-    """Check that the variable name is a valid Python variable name, and that
-    there is an equals sign. The value can by anything non-empty.
-    """
-    if not re.match(r"^[a-z_]\w*=.+$", value):
-        raise argparse.ArgumentTypeError(
-            "definition should be in var=value format: %r" % value
-        )
-    return value
-
-
 def dump_multi_config(conf_file, dest, **kwargs):
     """Given a multi compose config, clone it and all referenced files to a
     given directory.
@@ -156,7 +146,7 @@ def main():
         action="append",
         default=[],
         metavar="VAR=VALUE",
-        type=validate_definition,
+        type=config_utils.validate_definition,
         help=(
             "Define a variable on command line and inject it into the config file. "
             "Can be used multiple times."
@@ -185,7 +175,7 @@ def main():
 
     args = parser.parse_args()
 
-    defines = dict(var.split("=", 1) for var in args.define)
+    defines = config_utils.extract_defines(args.define)
 
     if args.multi:
         if len(args.sources) > 1:

bin/pungi-config-validate

@@ -22,6 +22,7 @@ import pungi.phases
 import pungi.wrappers.scm
 import pungi.util
 from pungi.wrappers.variants import VariantsXmlParser, VariantsValidationError
+from pungi_utils import config_utils
 
 
 class ValidationCompose(pungi.compose.Compose):
@@ -76,8 +77,23 @@ def read_variants(compose, config):
             compose.all_variants[child.uid] = child
 
 
-def run(config, topdir, has_old, offline):
+def run(config, topdir, has_old, offline, defined_variables):
-    conf = pungi.util.load_config(config)
+    # Load default values for undefined variables. This is useful for
+    # validating templates that are supposed to be filled in later with
+    # pungi-config-dump.
+    try:
+        defaults_file = os.path.join(
+            os.path.dirname(config), ".pungi-config-validate.json"
+        )
+        with open(defaults_file) as f:
+            defined_variables.update(json.load(f))
+    except IOError:
+        pass
+    # Load actual configuration
+    conf = pungi.util.load_config(config, defined_variables)
+    # Remove the dummy variables used for defaults.
+    for key in defined_variables:
+        del conf[key]
 
     errors, warnings = pungi.checks.validate(conf, offline=offline)
     if errors or warnings:
@@ -148,10 +164,23 @@ def main(args=None):
         action="store_true",
         help="Do not validate git references in URLs",
     )
+    parser.add_argument(
+        "-e",
+        "--define",
+        action="append",
+        default=[],
+        metavar="VAR=VALUE",
+        type=config_utils.validate_definition,
+        help=(
+            "Define a variable on command line and inject it into the config file. "
+            "Can be used multiple times."
+        ),
+    )
     opts = parser.parse_args(args)
+    defines = config_utils.extract_defines(opts.define)
 
     with pungi.util.temp_dir() as topdir:
-        errors = run(opts.config, topdir, opts.old_composes, opts.offline)
+        errors = run(opts.config, topdir, opts.old_composes, opts.offline, defines)
 
     for msg in errors:
         print(msg)

pungi/util.py

@@ -935,9 +935,10 @@ def iter_module_defaults(path):
                 yield mmddef
 
 
-def load_config(file_path):
+def load_config(file_path, defaults={}):
     """Open and load configuration file form .conf or .json file."""
     conf = kobo.conf.PyConfigParser()
+    conf.load_from_dict(defaults)
     if file_path.endswith(".json"):
         with open(file_path) as f:
             conf.load_from_dict(json.load(f))

pungi_utils/config_utils.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+
+import argparse
+import re
+
+
+def validate_definition(value):
+    """Check that the variable name is a valid Python variable name, and that
+    there is an equals sign. The value can by anything non-empty.
+    """
+    if not re.match(r"^[a-z_]\w*=.*$", value):
+        raise argparse.ArgumentTypeError(
+            "definition should be in var=value format: %r" % value
+        )
+    return value
+
+
+def extract_defines(args):
+    """Given an iterable of "key=value" strings, parse them into a dict."""
+    return dict(var.split("=", 1) for var in args)

tests/test_config_utils.py

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
+import argparse
+import os
+import sys
+
+from parameterized import parameterized
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from pungi_utils import config_utils
+
+
+class TestDefineHelpers(unittest.TestCase):
+    @parameterized.expand(
+        [
+            ([], {}),
+            (["foo=bar", "baz=quux"], {"foo": "bar", "baz": "quux"}),
+            (["foo="], {"foo": ""}),
+            (["foo==bar"], {"foo": "=bar"}),
+        ]
+    )
+    def test_extract_defines(self, input, expected):
+        self.assertEqual(config_utils.extract_defines(input), expected)
+
+    @parameterized.expand(["foo=bar", "foo=", "foo==bar"])
+    def test_validate_define_correct(self, value):
+        self.assertEqual(config_utils.validate_definition(value), value)
+
+    @parameterized.expand(["foo", "=", "=foo", "1=2"])
+    def test_validate_define_incorrect(self, value):
+        with self.assertRaises(argparse.ArgumentTypeError):
+            config_utils.validate_definition(value)