rpms/leapp
5
0
Fork 0
Code Issues Pull Requests Releases Activity
c9
leapp/SOURCES/0003-configs-implement-actor-configuration-support.patch
eabdullin e177ea78a8 Import from CS git
2024-11-25 09:10:02 +00:00

1576 lines
64 KiB
Diff
Raw Permalink Blame History

From b1539e2dba07f5c0a0d6b76d53bc344890200965 Mon Sep 17 00:00:00 2001
From: Toshio Kuratomi <tkuratom@redhat.com>
Date: Thu, 12 Sep 2024 13:25:55 -0700
Subject: [PATCH 3/6] configs: implement actor configuration support
This commit introduces multiple improvements and fixes for actor
configuration management in LEAPP, including configuration schema
support, API updates, validation improvements, and compatibility fixes.
- Actor Configuration:
- Add actor configuration support
- Introduce configuration schema attributes for LEAPP actors.
- Create an API to load and validate actor configs against the schemas
stored in actors.
- Provide a function to retrieve actor-specified configurations.
- Enable config directories at both repository-wide and actor-wide
levels.
- Add configs to `leapp.db`.
- Configuration Schema Support:
- Add `_is_config_sequence()` to validate sequences (lists, tuples) of
configuration fields.
- Add support for `StringMap` field types in `config_schemas`.
- Testing and Linting:
- Separate linting and unittests for better CI control.
- Dependency Management:
- Add `pyyaml` to requirements in `requirements.txt`, `setup.py`, and
spec files.
Some unit tests were also updated as they rely on `os.chdir` which
is problematic as the patch tries to correctly reverse any os.chdir
that is executed. Therefore, we mock `os.chdir` in the appropriate
unit tests.
JIRA: OAMG-8803
---
.github/workflows/unit-tests.yml | 12 +
etc/leapp/leapp.conf | 2 +
leapp/actors/__init__.py | 78 +++--
leapp/actors/config.py | 318 ++++++++++++++++++
leapp/configs/__init__.py | 0
leapp/configs/actor/__init__.py | 13 +
leapp/configs/common/__init__.py | 4 +
leapp/libraries/stdlib/api.py | 7 +
leapp/models/fields/__init__.py | 78 ++++-
leapp/repository/__init__.py | 22 +-
leapp/repository/actor_definition.py | 41 ++-
leapp/repository/definition.py | 5 +-
leapp/repository/scan.py | 16 +
leapp/utils/audit/__init__.py | 65 +++-
leapp/workflows/__init__.py | 1 +
packaging/leapp.spec | 2 +
requirements.txt | 1 +
res/container-tests/Containerfile.ubi10 | 13 -
res/container-tests/Containerfile.ubi10-lint | 35 ++
res/container-tests/Containerfile.ubi7 | 13 -
res/container-tests/Containerfile.ubi7-lint | 28 ++
res/container-tests/Containerfile.ubi8 | 13 -
res/container-tests/Containerfile.ubi8-lint | 31 ++
res/container-tests/Containerfile.ubi9 | 13 -
res/container-tests/Containerfile.ubi9-lint | 30 ++
res/schema/audit-layout.sql | 13 +-
.../0003-add-actor-configuration.sql | 16 +
setup.py | 2 +-
tests/scripts/test_actor_config_db.py | 90 +++++
tests/scripts/test_metadata.py | 5 +-
.../test_repository_actor_definition.py | 6 +-
31 files changed, 876 insertions(+), 97 deletions(-)
create mode 100644 leapp/actors/config.py
create mode 100644 leapp/configs/__init__.py
create mode 100644 leapp/configs/actor/__init__.py
create mode 100644 leapp/configs/common/__init__.py
create mode 100644 res/container-tests/Containerfile.ubi10-lint
create mode 100644 res/container-tests/Containerfile.ubi7-lint
create mode 100644 res/container-tests/Containerfile.ubi8-lint
create mode 100644 res/container-tests/Containerfile.ubi9-lint
create mode 100644 res/schema/migrations/0003-add-actor-configuration.sql
create mode 100644 tests/scripts/test_actor_config_db.py
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
index a0a3942..d994b27 100644
--- a/.github/workflows/unit-tests.yml
+++ b/.github/workflows/unit-tests.yml
@@ -18,15 +18,27 @@ jobs:
- name: Run unit tests with python3.12 on el9
python: python3.12
container: ubi10
+ - name: Run python linters with python3.12 on el9
+ python: python3.12
+ container: ubi10-lint
- name: Run unit tests with python3.9 on el9
python: python3.9
container: ubi9
+ - name: Run python linters with python3.9 on el9
+ python: python3.9
+ container: ubi9-lint
- name: Run unit tests with python 3.6 on el8
python: python3.6
container: ubi8
+ - name: Run python linters with python 3.6 on el8
+ python: python3.6
+ container: ubi8-lint
- name: Run unit tests with python2.7 on el7
python: python2.7
container: ubi7
+ - name: Run python linters with python2.7 on el7
+ python: python2.7
+ container: ubi7-lint
steps:
- name: Checkout code
diff --git a/etc/leapp/leapp.conf b/etc/leapp/leapp.conf
index d641b30..a96dcce 100644
--- a/etc/leapp/leapp.conf
+++ b/etc/leapp/leapp.conf
@@ -4,3 +4,5 @@ repo_path=/etc/leapp/repos.d/
[database]
path=/var/lib/leapp/leapp.db
+[actor_config]
+path=/etc/leapp/actor_conf.d/
diff --git a/leapp/actors/__init__.py b/leapp/actors/__init__.py
index 9d83bf1..5970bdb 100644
--- a/leapp/actors/__init__.py
+++ b/leapp/actors/__init__.py
@@ -1,7 +1,16 @@
+import functools
import logging
import os
import sys
+try:
+ # Python 3.3+
+ from collections.abc import Sequence
+except ImportError:
+ # Python 2.6 through 3.2
+ from collections import Sequence
+
+from leapp.actors.config import Config, retrieve_config
from leapp.compat import string_types
from leapp.dialogs import Dialog
from leapp.exceptions import (MissingActorAttributeError, RequestStopAfterPhase, StopActorExecution,
@@ -41,6 +50,11 @@ class Actor(object):
Write the actor's description as a docstring.
"""
+ config_schemas = ()
+ """
+ Defines the structure of the configuration that the actor uses.
+ """
+
consumes = ()
"""
Tuple of :py:class:`leapp.models.Model` derived classes defined in the :ref:`repositories <terminology:repository>`
@@ -86,6 +100,7 @@ class Actor(object):
'path': os.path.dirname(sys.modules[type(self).__module__].__file__),
'class_name': type(self).__name__,
'description': self.description or type(self).__doc__,
+ 'config_schemas': [c.__name__ for c in self.config_schemas],
'consumes': [c.__name__ for c in self.consumes],
'produces': [p.__name__ for p in self.produces],
'tags': [t.__name__ for t in self.tags],
@@ -100,6 +115,7 @@ class Actor(object):
This depends on the definition of such a configuration model being defined by the workflow
and an actor that provides such a message.
"""
+
Actor.current_instance = self
install_translation_for_actor(type(self))
self._messaging = messaging
@@ -107,8 +123,12 @@ class Actor(object):
self.skip_dialogs = skip_dialogs
""" A configured logger instance for the current actor. """
+ # self._configuration is the workflow configuration.
+ # self.config_schemas is the actor defined configuration.
+ # self.config is the actual actor configuration
if config_model:
self._configuration = next(self.consume(config_model), None)
+ self.config = retrieve_config(self.config_schemas)
self._path = path
@@ -359,6 +379,15 @@ class Actor(object):
actor=self,
details=details)
+ def retrieve_config(self):
+ """
+ Retrieve the configuration described by self.config_schema.
+
+ :return: Dictionary containing requested configuration.
+ :rtype: dict
+ """
+ return retrieve_config(self.config_schema)
+
def _is_type(value_type):
def validate(actor, name, value):
@@ -390,17 +419,23 @@ def _lint_warn(actor, name, type_name):
logging.getLogger("leapp.linter").warning("Actor %s field %s should be a tuple of %s", actor, name, type_name)
-def _is_model_tuple(actor, name, value):
- if isinstance(value, type) and issubclass(value, Model):
- _lint_warn(actor, name, "Models")
+def _is_foo_sequence(cls, cls_name, actor, name, value):
+ if isinstance(value, type) and issubclass(value, cls):
+ _lint_warn(actor, name, cls_name)
value = (value,)
- _is_type(tuple)(actor, name, value)
- if not all([True] + [isinstance(item, type) and issubclass(item, Model) for item in value]):
+ _is_type(Sequence)(actor, name, value)
+ if not all(isinstance(item, type) and issubclass(item, cls) for item in value):
raise WrongAttributeTypeError(
- 'Actor {} attribute {} should contain only Models'.format(actor, name))
+ 'Actor {} attribute {} should contain only {}'.format(actor, name, cls_name))
return value
+_is_config_sequence = functools.partial(_is_foo_sequence, Config, "Configs")
+_is_model_sequence = functools.partial(_is_foo_sequence, Model, "Models")
+_is_tag_sequence = functools.partial(_is_foo_sequence, Tag, "Tags")
+_is_api_sequence = functools.partial(_is_foo_sequence, WorkflowAPI, "WorkflowAPIs")
+
+
def _is_dialog_tuple(actor, name, value):
if isinstance(value, Dialog):
_lint_warn(actor, name, "Dialogs")
@@ -412,28 +447,6 @@ def _is_dialog_tuple(actor, name, value):
return value
-def _is_tag_tuple(actor, name, value):
- if isinstance(value, type) and issubclass(value, Tag):
- _lint_warn(actor, name, "Tags")
- value = (value,)
- _is_type(tuple)(actor, name, value)
- if not all([True] + [isinstance(item, type) and issubclass(item, Tag) for item in value]):
- raise WrongAttributeTypeError(
- 'Actor {} attribute {} should contain only Tags'.format(actor, name))
- return value
-
-
-def _is_api_tuple(actor, name, value):
- if isinstance(value, type) and issubclass(value, WorkflowAPI):
- _lint_warn(actor, name, "Apis")
- value = (value,)
- _is_type(tuple)(actor, name, value)
- if not all([True] + [isinstance(item, type) and issubclass(item, WorkflowAPI) for item in value]):
- raise WrongAttributeTypeError(
- 'Actor {} attribute {} should contain only WorkflowAPIs'.format(actor, name))
- return value
-
-
def _get_attribute(actor, name, validator, required=False, default_value=None, additional_info='', resolve=None):
if resolve:
value = resolve(actor, name)
@@ -464,13 +477,14 @@ def get_actor_metadata(actor):
# # if path is not transformed into the realpath.
('path', os.path.dirname(os.path.realpath(sys.modules[actor.__module__].__file__))),
_get_attribute(actor, 'name', _is_type(string_types), required=True),
- _get_attribute(actor, 'tags', _is_tag_tuple, required=True, additional_info=additional_tag_info),
- _get_attribute(actor, 'consumes', _is_model_tuple, required=False, default_value=(), resolve=get_api_models),
- _get_attribute(actor, 'produces', _is_model_tuple, required=False, default_value=(), resolve=get_api_models),
+ _get_attribute(actor, 'tags', _is_tag_sequence, required=True, additional_info=additional_tag_info),
+ _get_attribute(actor, 'consumes', _is_model_sequence, required=False, default_value=(), resolve=get_api_models),
+ _get_attribute(actor, 'produces', _is_model_sequence, required=False, default_value=(), resolve=get_api_models),
_get_attribute(actor, 'dialogs', _is_dialog_tuple, required=False, default_value=()),
_get_attribute(actor, 'description', _is_type(string_types), required=False,
default_value=actor.__doc__ or 'There has been no description provided for this actor.'),
- _get_attribute(actor, 'apis', _is_api_tuple, required=False, default_value=())
+ _get_attribute(actor, 'config_schemas', _is_config_sequence, required=False, default_value=()),
+ _get_attribute(actor, 'apis', _is_api_sequence, required=False, default_value=())
])
diff --git a/leapp/actors/config.py b/leapp/actors/config.py
new file mode 100644
index 0000000..160c159
--- /dev/null
+++ b/leapp/actors/config.py
@@ -0,0 +1,318 @@
+"""
+Config file format:
+ yaml file like this:
+
+---
+# Note: have to add a fields.Map type before we can use yaml mappings.
+section_name:
+ field1_name: value
+ field2_name:
+ - listitem1
+ - listitem2
+section2_name:
+ field3_name: value
+
+Config files are any yaml files in /etc/leapp/actor_config.d/
+(This is settable in /etc/leapp/leapp.conf)
+
+"""
+__metaclass__ = type
+
+import abc
+import glob
+import logging
+import os.path
+from collections import defaultdict
+
+import six
+import yaml
+
+from leapp.models.fields import ModelViolationError
+
+
+try:
+ # Compiled versions if available, for speed
+ from yaml import CSafeLoader as SafeLoader
+except ImportError:
+ from yaml import SafeLoader
+
+
+_ACTOR_CONFIG = None
+
+log = logging.getLogger('leapp.actors.config')
+
+
+class SchemaError(Exception):
+ """
+ Raised when actors use conflicting schemas.
+
+ For example, one schema wants `section.field` to be an integer and other schema wants
+ `section.field` to be a string.
+ """
+
+
+class ValidationError(Exception):
+ """
+ Raised when a config file fails to validate against any of the available schemas.
+ """
+
+
+# pylint: disable=deprecated-decorator
+# @abc.abstractproperty is deprecated in newer Python3 versions but it's
+# necessary for Python <= 3.3 (including 2.7)
+@six.add_metaclass(abc.ABCMeta)
+class Config:
+ """
+ An Actor config schema looks like this.
+
+ ::
+ class RHUIConfig(Config):
+ section = "rhui"
+ name = "file_map"
+ type_ = fields.Map(fields.String())
+ description = 'Description here'
+ default = {"repo": "url"}
+ """
+ @abc.abstractproperty
+ def section(self):
+ pass
+
+ @abc.abstractproperty
+ def name(self):
+ pass
+
+ @abc.abstractproperty
+ def type_(self):
+ pass
+
+ @abc.abstractproperty
+ def description(self):
+ pass
+
+ @abc.abstractproperty
+ def default(self):
+ pass
+
+ @classmethod
+ def to_dict(cls):
+ """
+ Return a dictionary representation of the config item that would be suitable for putting
+ into a config file.
+ """
+ representation = {
+ cls.section: {
+ '{0}_description__'.format(cls.name): cls.description
+ }
+ }
+ # TODO: Retrieve the default values from the type field.
+ # Something like this maybe:
+ # representation[cls.section][cls.name] = cls.type_.get_default()
+
+ return representation
+
+ @classmethod
+ def serialize(cls):
+ """
+ :return: Serialized information for the config
+ """
+ return {
+ 'class_name': cls.__name__,
+ 'section': cls.section,
+ 'name': cls.name,
+ 'type': cls.type_.serialize(),
+ 'description': cls.description,
+ 'default': cls.default,
+ }
+# pylint: enable=deprecated-decorator
+
+
+def _merge_config(configuration, new_config):
+ """
+ Merge two dictionaries representing configuration. fields in new_config overwrite
+ any existing fields of the same name in the same section in configuration.
+ """
+ for section_name, section in new_config.items():
+ if section_name not in configuration:
+ configuration[section_name] = section
+ else:
+ for field_name, field in section:
+ configuration[section_name][field_name] = field
+
+
+def _get_config(config_dir='/etc/leapp/actor_conf.d'):
+ """
+ Read all configuration files from the config_dir and return a dict with their values.
+ """
+ # We do not do a recursive walk to maintain Python2 compatibility, but we do
+ # not expect rich config hierarchies, so no harm done.
+ config_files = glob.glob(os.path.join(config_dir, '*'))
+ config_files = [f for f in config_files if f.endswith('.yml') or f.endswith('.yaml')]
+ config_files.sort()
+
+ configuration = {}
+ for config_file in config_files:
+ with open(config_file) as f:
+ raw_cfg = f.read()
+
+ try:
+ parsed_config = yaml.load(raw_cfg, SafeLoader)
+ except Exception as e:
+ log.warning("Warning: unparsable yaml file %s in the config directory."
+ " Error: %s", config_file, str(e))
+ raise
+
+ _merge_config(configuration, parsed_config)
+
+ return configuration
+
+
+def normalize_schemas(schemas):
+ """
+ Merge all schemas into a single dictionary and validate them for errors we can detect.
+ """
+ added_fields = set()
+ normalized_schema = defaultdict(dict)
+ for schema in schemas:
+ for field in schema:
+ unique_name = (field.section, field.name)
+
+ # Error if the field has been added by another schema
+ if unique_name in added_fields and added_fields[unique_name] != field:
+ # TODO: Also include information on what Actor contains the
+ # conflicting fields but that information isn't passed into
+ # this function right now.
+ message = ('Two actors added incompatible configuration items'
+ ' with the same name for Section: {section},'
+ ' Field: {field}'.format(section=field.section,
+ field=field.name))
+ log.error(message)
+ raise SchemaError(message)
+
+ # TODO: More validation here.
+
+ # Store the fields from the schema in a way that we can easily look
+ # up while validating
+ added_fields.add(unique_name)
+ normalized_schema[field.section][field.name] = field
+
+ return normalized_schema
+
+
+def _validate_field_type(field_type, field_value, field_path):
+ """
+ Return False if the field is not of the proper type.
+
+ :param str field_path: Path in the config where the field is placed.
+ Example: A field 'target_clients' in a section 'rhui' would have a path 'rhui.target_clients'
+ """
+ try:
+ # the name= parameter is displayed in error messages to let the user know what precisely is wrong
+ field_type._validate_model_value(field_value, name=field_path)
+ except ModelViolationError as e:
+ # Any problems mean that the field did not validate.
+ log.info("Configuration value failed to validate with: %s", e)
+ return False
+ return True
+
+
+def _normalize_config(actor_config, schema):
+ # Go through the config and log warnings about unexpected sections/fields.
+ for section_name, section in actor_config.items():
+ if section_name not in schema:
+ # TODO: Also have information about which config file contains the unknown field.
+ message = "A config file contained an unknown section: {section}".format(section=section_name)
+ log.warning(message)
+ continue
+
+ for field_name in actor_config:
+ # Any field names which end in "__" are reserved for LEAPP to use
+ # for its purposes. In particular, it places documentation of
+ # a field's value into these reserved field names.
+ if field_name.endswith("__"):
+ continue
+
+ if field_name not in schema[section_name]:
+ # TODO: Also have information about which config file contains the unknown field.
+ message = ("A config file contained an unknown field: (Section:"
+ " {section}, Field: {field})".format(
+ section=section_name, field=field_name)
+ )
+ log.warning(message)
+
+ # Do several things:
+ # * Validate that the config values are of the proper types.
+ # * Add default values where no config value was provided.
+ normalized_actor_config = {}
+ for section_name, section in schema.items():
+ for field_name, field in section.items():
+ # TODO: We might be able to do this using the default piece of
+ # model.fields.Field(). Something using
+ # schema[section_name, field_name].type_ with the value from
+ # actor_config[section_name][field_name]. But looking at the Model
+ # code, I wasn't quite sure how this should be done so I think this
+ # will work for now.
+
+ # For every item in the schema, either retrieve the value from the
+ # config files or set it to the default.
+ try:
+ value = actor_config[section_name][field_name]
+ except KeyError:
+ # Either section_name or field_name doesn't exist
+ section = actor_config[section_name] = actor_config.get(section_name, {})
+ # May need to deepcopy default if these values are modified.
+ # However, it's probably an error if they are modified and we
+ # should possibly look into disallowing that.
+ value = field.default
+ section[field_name] = value
+
+ field_path = '{0}.{1}'.format(section_name, field_name)
+ if not _validate_field_type(field.type_, value, field_path):
+ raise ValidationError("Config value for (Section: {section},"
+ " Field: {field}) is not of the correct"
+ " type".format(section=section_name,
+ field=field_name)
+ )
+
+ normalized_section = normalized_actor_config.get(section_name, {})
+ normalized_section[field_name] = value
+ # If the section already exists, this is a no-op. Otherwise, it
+ # sets it to the newly created dict.
+ normalized_actor_config[section_name] = normalized_section
+
+ return normalized_actor_config
+
+
+def load(config_dir, schemas):
+ """
+ Return Actor Configuration.
+
+ :returns: a dict representing the configuration.
+ :raises ValueError: if the actor configuration does not match the schema.
+
+ This function reads the config, validates it, and adds any default values.
+ """
+ global _ACTOR_CONFIG
+ if _ACTOR_CONFIG:
+ return _ACTOR_CONFIG
+
+ config = _get_config(config_dir)
+ config = _normalize_config(config, schemas)
+
+ _ACTOR_CONFIG = config
+ return _ACTOR_CONFIG
+
+
+def retrieve_config(schema):
+ """Called by the actor to retrieve the actor configuration specific to this actor."""
+ # TODO: The use of _ACTOR_CONFIG isn't good API. Since this function is
+ # called by the Actors, we *know* that this is okay to do (as the
+ # configuration will have already been loaded.) However, there's nothing in
+ # the API that ensures that this is the case. Need to redesign this.
+ # Can't think of how it should look right now because loading requires
+ # information that the Actor doesn't know.
+
+ configuration = defaultdict(dict)
+ for field in schema:
+ configuration[field.section][field.name] = _ACTOR_CONFIG[field.section][field.name]
+
+ return dict(configuration)
diff --git a/leapp/configs/__init__.py b/leapp/configs/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/leapp/configs/actor/__init__.py b/leapp/configs/actor/__init__.py
new file mode 100644
index 0000000..37a0c80
--- /dev/null
+++ b/leapp/configs/actor/__init__.py
@@ -0,0 +1,13 @@
+"""
+:py:mod:`leapp.configs.actor` represents the import location for private actor config schema that
+are placed in the actor's configs folder.
+
+
+Example:
+ If your actor has a configs folder with a schemas.py python module, import it
+ from the actor like this::
+
+ from leapp.configs.actor import schemas
+
+This directory is intended for the definitions of actor configuration fields.
+"""
diff --git a/leapp/configs/common/__init__.py b/leapp/configs/common/__init__.py
new file mode 100644
index 0000000..d41c86c
--- /dev/null
+++ b/leapp/configs/common/__init__.py
@@ -0,0 +1,4 @@
+"""
+:py:mod:`leapp.configs.common` represents an import location for shared libraries that
+are placed in the repository's configs folder.
+"""
diff --git a/leapp/libraries/stdlib/api.py b/leapp/libraries/stdlib/api.py
index c49a03d..267468b 100644
--- a/leapp/libraries/stdlib/api.py
+++ b/leapp/libraries/stdlib/api.py
@@ -228,3 +228,10 @@ def get_actor_tool_path(name):
:rtype: str or None
"""
return current_actor().get_actor_tool_path(name)
+
+
+def retrieve_config():
+ """
+ Retrieve the configuration specific to the specified schema.
+ """
+ return current_actor().retrieve_config()
diff --git a/leapp/models/fields/__init__.py b/leapp/models/fields/__init__.py
index 5e24be4..e5052c6 100644
--- a/leapp/models/fields/__init__.py
+++ b/leapp/models/fields/__init__.py
@@ -2,6 +2,12 @@ import base64
import copy
import datetime
import json
+try:
+ # Python 3
+ from collections.abc import Sequence
+except ImportError:
+ # Python 2.7
+ from collections import Sequence
import six
@@ -147,7 +153,7 @@ class Field(object):
def serialize(self):
"""
- :return: Serialized form of the workflow
+ :return: Serialized form of the field
"""
return {
'nullable': self._nullable,
@@ -185,11 +191,13 @@ class BuiltinField(Field):
self._validate(value=value, name=name, expected_type=self._builtin_type)
def _validate(self, value, name, expected_type):
- if not isinstance(expected_type, tuple):
+ if not isinstance(expected_type, Sequence):
expected_type = (expected_type,)
+
if value is None and self._nullable:
return
- if not any(isinstance(value, t) for t in expected_type):
+
+ if not isinstance(value, expected_type):
names = ', '.join(['{}'.format(t.__name__) for t in expected_type])
raise ModelViolationError("Fields {} is of type: {} expected: {}".format(name, type(value).__name__,
names))
@@ -457,6 +465,70 @@ class List(Field):
return result
+class StringMap(Field):
+ """
+ Map from strings to instances of a given value type.
+ """
+ def __init__(self, value_type, **kwargs):
+ super(StringMap, self).__init__(**kwargs)
+
+ if self._default is not None:
+ self._default = copy.copy(self._default)
+
+ if not isinstance(value_type, Field):
+ raise ModelMisuseError("value_type must be an instance of a type derived from Field")
+
+ self._value_type = value_type
+
+ def _validate_model_value_using_validator(self, new_map, name, validation_method):
+ list_validator_fn = getattr(super(StringMap, self), validation_method)
+ list_validator_fn(new_map, name)
+
+ if isinstance(new_map, dict):
+ for key in new_map:
+ # Check that the key is trully a string
+ if not isinstance(key, str):
+ err = 'Expected a key of type `str`, but got a key `{}` of type `{}`'
+ raise ModelViolationError(err.format(key, type(key).__name__))
+
+ value = new_map[key] # avoid using .items(), as it creates a list of all items (slow) in py2
+
+ # _value_type's validation will check whether the value has a correct type
+ value_validator_fn = getattr(self._value_type, validation_method)
+ value_validator_fn(value, name='{}[{}]'.format(name, key))
+ elif value is not None:
+ raise ModelViolationError('Expected a dict but got {} for the {} field'.format(type(value).__name__, name))
+
+ def _validate_model_value(self, value, name):
+ self._validate_model_value_using_validator(value, name, '_validate_model_value')
+
+ def _validate_builtin_value(self, value, name):
+ self._validate_model_value_using_validator(value, name, '_validate_builtin_value')
+
+ def _convert_to_model(self, value, name):
+ self._validate_builtin_value(value=value, name=name)
+
+ if value is None:
+ return value
+
+ converter = self._value_type._convert_to_model
+ return {key: converter(value[key], name='{0}["{1}"]'.format(name, key)) for key in value}
+
+ def _convert_from_model(self, value, name):
+ self._validate_model_value(value=value, name=name)
+
+ if value is None:
+ return value
+
+ converter = self._value_type._convert_from_model
+ return {key: converter(value[key], name='{0}["{1}"]'.format(name, key)) for key in value}
+
+ def serialize(self):
+ result = super(StringMap, self).serialize()
+ result['value_type'] = self._value_type.serialize()
+ return result
+
+
class Model(Field):
"""
Model is used to use other Models as fields
diff --git a/leapp/repository/__init__.py b/leapp/repository/__init__.py
index 58b6d16..8b7aac4 100644
--- a/leapp/repository/__init__.py
+++ b/leapp/repository/__init__.py
@@ -7,6 +7,7 @@ from leapp.compat import load_module
from leapp.exceptions import RepoItemPathDoesNotExistError, UnsupportedDefinitionKindError
from leapp.models import get_models, resolve_model_references
import leapp.libraries.common # noqa # pylint: disable=unused-import
+import leapp.configs.common # noqa # pylint: disable=unused-import
from leapp.repository.actor_definition import ActorDefinition
from leapp.repository.definition import DefinitionKind
from leapp.tags import get_tags
@@ -144,8 +145,10 @@ class Repository(object):
self.log.debug("Extending LEAPP_COMMON_FILES for common file paths")
self._extend_environ_paths('LEAPP_COMMON_FILES', self.files)
self.log.debug("Installing repository provided common libraries loader hook")
+
sys.meta_path.append(LeappLibrariesFinder(module_prefix='leapp.libraries.common', paths=self.libraries))
sys.meta_path.append(LeappLibrariesFinder(module_prefix='leapp.workflows.api', paths=self.apis))
+ sys.meta_path.append(LeappLibrariesFinder(module_prefix='leapp.configs.common', paths=self.configs))
if not skip_actors_discovery:
if not stage or stage is _LoadStage.ACTORS:
@@ -189,6 +192,15 @@ class Repository(object):
})
return data
+ # Note: `configs` are not present here because we are not yet making
+ # them globally accessible. This is to force people to copy the config
+ # schema to their Actors instead of importing them from other Actors.
+ # That copy, in turn, is a good idea so the framework can return an
+ # error if two Actors share the same config but they have different
+ # schemafor it (for instance, if Actor Foo and Bar were sharing the
+ # same config entry but Actor Foo updated the entry in a later version.
+ # We need to error so Actor Bar can either be ported to the new
+ # definition or use a different config entry for it's needs.)
return {
'repo_dir': self._repo_dir,
'actors': [mapped_actor_data(a.serialize()) for a in self.actors],
@@ -199,7 +211,8 @@ class Repository(object):
'workflows': filtered_serialization(get_workflows, self.workflows),
'tools': [dict([('path', path)]) for path in self.relative_paths(self.tools)],
'files': [dict([('path', path)]) for path in self.relative_paths(self.files)],
- 'libraries': [dict([('path', path)]) for path in self.relative_paths(self.libraries)]
+ 'libraries': [dict([('path', path)]) for path in self.relative_paths(self.libraries)],
+ 'configs': [dict([('path', path)]) for path in self.relative_paths(self.configs)],
}
def _extend_environ_paths(self, name, paths):
@@ -268,6 +281,13 @@ class Repository(object):
"""
return tuple(self._definitions.get(DefinitionKind.LIBRARIES, ()))
+ @property
+ def configs(self):
+ """
+ :return: Tuple of configs in the repository
+ """
+ return tuple(self._definitions.get(DefinitionKind.CONFIGS, ()))
+
@property
def files(self):
"""
diff --git a/leapp/repository/actor_definition.py b/leapp/repository/actor_definition.py
index 0e9cc0a..454d7ad 100644
--- a/leapp/repository/actor_definition.py
+++ b/leapp/repository/actor_definition.py
@@ -175,6 +175,7 @@ class ActorDefinition(object):
'class_name': self.class_name,
'description': self.description,
'tags': self.tags,
+ 'config_schemas': self.config_schemas,
'consumes': self.consumes,
'produces': self.produces,
'apis': self.apis,
@@ -182,6 +183,7 @@ class ActorDefinition(object):
'tools': self.tools,
'files': self.files,
'libraries': self.libraries,
+ 'configs': self.configs,
'tests': self.tests
}
@@ -212,7 +214,21 @@ class ActorDefinition(object):
if p.exitcode != 0:
self.log.error("Process inspecting actor in %s failed with %d", self.directory, p.exitcode)
raise ActorInspectionFailedError('Inspection of actor in {path} failed'.format(path=self.directory))
- result = q.get()
+
+ # Note (dkubek): The use of injected_context() here is necessary
+ # when retrieving the discovered actor. This is because
+ # configuration schemas are defined within the actor's scope, and
+ # how Leapp manages imports. Configuration schemas, being scoped
+ # within each actor, are accessible internally by the actor itself.
+ # However, attempting to return a configuration schema outside this
+ # scope results in an error. This occurs because the schema class
+ # definition is no longer accessible in the path, making it
+ # unavailable for importint. The injected_context() here ensures
+ # the path to the actor level code remains in scope while accessing
+ # it.
+ with self.injected_context():
+ result = q.get()
+
if not result:
self.log.error("Process inspecting actor in %s returned no result", self.directory)
raise ActorInspectionFailedError(
@@ -279,6 +295,13 @@ class ActorDefinition(object):
"""
return self.discover()['description']
+ @property
+ def config_schemas(self):
+ """
+ :return: Actor config_schemas
+ """
+ return self.discover()['config_schemas']
+
@contextlib.contextmanager
def injected_context(self):
"""
@@ -301,11 +324,18 @@ class ActorDefinition(object):
if self.tools:
os.environ['LEAPP_TOOLS'] = os.path.join(self._repo_dir, self._directory, self.tools[0])
+ meta_path_backup = sys.meta_path[:]
+
sys.meta_path.append(
LeappLibrariesFinder(
module_prefix='leapp.libraries.actor',
paths=[os.path.join(self._repo_dir, self.directory, x) for x in self.libraries]))
+ sys.meta_path.append(
+ LeappLibrariesFinder(
+ module_prefix='leapp.configs.actor',
+ paths=[os.path.join(self._repo_dir, self.directory, x) for x in self.configs]))
+
previous_path = os.getcwd()
os.chdir(os.path.join(self._repo_dir, self._directory))
try:
@@ -326,6 +356,8 @@ class ActorDefinition(object):
else:
os.environ.pop('LEAPP_TOOLS', None)
+ sys.meta_path = meta_path_backup
+
@property
def apis(self):
"""
@@ -361,6 +393,13 @@ class ActorDefinition(object):
"""
return tuple(self._definitions.get(DefinitionKind.FILES, ()))
+ @property
+ def configs(self):
+ """
+ :return: Tuple with path to the configs folder of the actor, empty tuple if none
+ """
+ return tuple(self._definitions.get(DefinitionKind.CONFIGS, ()))
+
@property
def tests(self):
"""
diff --git a/leapp/repository/definition.py b/leapp/repository/definition.py
index e853a6a..344ec77 100644
--- a/leapp/repository/definition.py
+++ b/leapp/repository/definition.py
@@ -14,8 +14,9 @@ class DefinitionKind(object):
LIBRARIES = _Kind('libraries')
TOOLS = _Kind('tools')
FILES = _Kind('files')
+ CONFIGS = _Kind('configs')
TESTS = _Kind('tests')
API = _Kind('api')
- REPO_WHITELIST = (ACTOR, API, MODEL, TOPIC, TAG, WORKFLOW, TOOLS, LIBRARIES, FILES)
- ACTOR_WHITELIST = (TOOLS, LIBRARIES, FILES, TESTS)
+ REPO_WHITELIST = (ACTOR, API, MODEL, TOPIC, TAG, WORKFLOW, TOOLS, LIBRARIES, FILES, CONFIGS)
+ ACTOR_WHITELIST = (TOOLS, LIBRARIES, FILES, CONFIGS, TESTS)
diff --git a/leapp/repository/scan.py b/leapp/repository/scan.py
index 378940a..94e0512 100644
--- a/leapp/repository/scan.py
+++ b/leapp/repository/scan.py
@@ -89,6 +89,7 @@ def scan(repository, path):
('workflows', scan_workflows),
('files', scan_files),
('libraries', scan_libraries),
+ ('configs', scan_configs),
('tests', scan_tests),
('tools', scan_tools),
('apis', scan_apis))
@@ -224,6 +225,21 @@ def scan_libraries(repo, path, repo_path):
repo.add(DefinitionKind.LIBRARIES, os.path.relpath(path, repo_path))
+def scan_configs(repo, path, repo_path):
+ """
+ Scans configs and adds them to the repository.
+
+ :param repo: Instance of the repository
+ :type repo: :py:class:`leapp.repository.Repository`
+ :param path: path to the configs
+ :type path: str
+ :param repo_path: path to the repository
+ :type repo_path: str
+ """
+ if os.listdir(path):
+ repo.add(DefinitionKind.CONFIGS, os.path.relpath(path, repo_path))
+
+
def scan_tools(repo, path, repo_path):
"""
Scans tools and adds them to the repository.
diff --git a/leapp/utils/audit/__init__.py b/leapp/utils/audit/__init__.py
index 16db107..8511179 100644
--- a/leapp/utils/audit/__init__.py
+++ b/leapp/utils/audit/__init__.py
@@ -222,6 +222,67 @@ class DataSource(Host):
self._data_source_id = cursor.fetchone()[0]
+class ActorConfigData(Storable):
+ """
+ Actor configuration data
+ """
+
+ def __init__(self, config=None, hash_id=None):
+ """
+ :param config: Actor configuration
+ :type config: str
+ :param hash_id: SHA256 hash in hexadecimal representation of config
+ :type hash_id: str
+ """
+ super(ActorConfigData, self).__init__()
+ self.config = config
+ self.hash_id = hash_id
+
+ def do_store(self, connection):
+ super(ActorConfigData, self).do_store(connection)
+ connection.execute(
+ 'INSERT OR IGNORE INTO actor_config_data (hash, config) VALUES(?, ?)',
+ (self.hash_id, self.config)
+ )
+
+
+class ActorConfig(Storable):
+ """
+ Actor configuration
+ """
+
+ def __init__(self, context=None, config=None):
+ """
+ :param context: The execution context
+ :type context: str
+ :param config: Actor configuration
+ :type config: :py:class:`leapp.utils.audit.ActorConfigData`
+ """
+ super(ActorConfig, self).__init__()
+ self.context = context
+ self.config = config
+ self._actor_config_id = None
+
+ @property
+ def actor_config_id(self):
+ """
+ Returns the id of the entry, which is only set when already stored.
+ :return: Integer id or None
+ """
+ return self._actor_config_id
+
+ def do_store(self, connection):
+ super(ActorConfig, self).do_store(connection)
+ self.config.do_store(connection)
+ connection.execute(
+ 'INSERT OR IGNORE INTO actor_config (context, actor_config_hash) VALUES(?, ?)',
+ (self.context, self.config.hash_id))
+ cursor = connection.execute(
+ 'SELECT id FROM actor_config WHERE context = ? AND actor_config_hash = ?',
+ (self.context, self.config.hash_id))
+ self._actor_config_id = cursor.fetchone()[0]
+
+
class Metadata(Storable):
"""
Metadata of an Entity
@@ -651,10 +712,12 @@ def store_actor_metadata(actor_definition, phase):
'consumes': sorted(model.__name__ for model in _metadata.get('consumes', ())),
'produces': sorted(model.__name__ for model in _metadata.get('produces', ())),
'tags': sorted(tag.__name__ for tag in _metadata.get('tags', ())),
+ 'config_schemas': [field.serialize() for field in _metadata.get('config_schemas', ())],
})
_metadata['phase'] = phase
- actor_metadata_fields = ('class_name', 'name', 'description', 'phase', 'tags', 'consumes', 'produces', 'path')
+ actor_metadata_fields = ('class_name', 'name', 'description', 'phase',
+ 'tags', 'consumes', 'produces', 'config_schemas', 'path')
metadata = json.dumps({field: _metadata[field] for field in actor_metadata_fields}, sort_keys=True)
metadata_hash_id = hashlib.sha256(metadata.encode('utf-8')).hexdigest()
diff --git a/leapp/workflows/__init__.py b/leapp/workflows/__init__.py
index 1b6fc98..b6168ee 100644
--- a/leapp/workflows/__init__.py
+++ b/leapp/workflows/__init__.py
@@ -4,6 +4,7 @@ import socket
import sys
import uuid
+from leapp.actors.config import retrieve_config
from leapp.dialogs import RawMessageDialog
from leapp.exceptions import CommandError, MultipleConfigActorsError, WorkflowConfigNotAvailable
from leapp.messaging.answerstore import AnswerStore
diff --git a/packaging/leapp.spec b/packaging/leapp.spec
index ab38d20..29ece53 100644
--- a/packaging/leapp.spec
+++ b/packaging/leapp.spec
@@ -134,6 +134,7 @@ Provides: leapp-framework-dependencies = %{framework_dependencies}
Requires: python-six
Requires: python-setuptools
Requires: python-requests
+Requires: PyYAML
%else # <> rhel 7
# for Fedora & RHEL 8+ deliver just python3 stuff
# NOTE: requirement on python3 refers to the general version of Python
@@ -144,6 +145,7 @@ Requires: python3
Requires: python3-six
Requires: python3-setuptools
Requires: python3-requests
+Requires: python3-PyYAML
%endif
Requires: findutils
##################################################
diff --git a/requirements.txt b/requirements.txt
index ec831ad..2951f23 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1,3 @@
six==1.12
requests
+PyYAML
diff --git a/res/container-tests/Containerfile.ubi10 b/res/container-tests/Containerfile.ubi10
index 6d6507c..1d15d29 100644
--- a/res/container-tests/Containerfile.ubi10
+++ b/res/container-tests/Containerfile.ubi10
@@ -20,19 +20,6 @@ ENTRYPOINT virtualenv testenv -p "/usr/bin/$PYTHON_VENV" && \
pip install -U pytest && \
pip install -U six && \
pip install -U . && \
- export LINTABLES=$(find . -name '*.py' | grep -E -e '^\./leapp\/' -e '^\./tests/scripts/' | sort -u ) && \
- echo '==================================================' && \
- echo '==================================================' && \
- echo '=============== Running pylint ===============' && \
- echo '==================================================' && \
- echo '==================================================' && \
- echo $LINTABLES | xargs pylint && echo '===> pylint PASSED' && \
- echo '==================================================' && \
- echo '==================================================' && \
- echo '=============== Running flake8 ===============' && \
- echo '==================================================' && \
- echo '==================================================' && \
- flake8 $LINTABLES && echo '===> flake8 PASSED' && \
echo '==================================================' && \
echo '==================================================' && \
echo '=============== Running tests ===============' && \
diff --git a/res/container-tests/Containerfile.ubi10-lint b/res/container-tests/Containerfile.ubi10-lint
new file mode 100644
index 0000000..a32e440
--- /dev/null
+++ b/res/container-tests/Containerfile.ubi10-lint
@@ -0,0 +1,35 @@
+FROM registry.access.redhat.com/ubi9/ubi:latest
+
+VOLUME /payload
+
+RUN dnf update -y && \
+ dnf install python3 python312 python3-pip make -y && \
+ python3 -m pip install --upgrade pip==20.3.4
+
+RUN pip install virtualenv
+
+WORKDIR /payload
+ENTRYPOINT virtualenv testenv -p "/usr/bin/$PYTHON_VENV" && \
+ source testenv/bin/activate && \
+ pip install -U setuptools && \
+ pip install -U funcsigs && \
+ pip install -U -r requirements-tests.txt && \
+ # NOTE(mmatuska): The pytest ver defined in requirements-tests is too old \
+ # for Python 3.12 (missing imp module), there do an update here until we \
+ # bump that. Similarly for six. \
+ pip install -U pytest && \
+ pip install -U six && \
+ pip install -U . && \
+ export LINTABLES=$(find . -name '*.py' | grep -E -e '^\./leapp\/' -e '^\./tests/scripts/' | sort -u ) && \
+ echo '==================================================' && \
+ echo '==================================================' && \
+ echo '=============== Running pylint ===============' && \
+ echo '==================================================' && \
+ echo '==================================================' && \
+ echo $LINTABLES | xargs pylint && echo '===> pylint PASSED' && \
+ echo '==================================================' && \
+ echo '==================================================' && \
+ echo '=============== Running flake8 ===============' && \
+ echo '==================================================' && \
+ echo '==================================================' && \
+ flake8 $LINTABLES && echo '===> flake8 PASSED'
diff --git a/res/container-tests/Containerfile.ubi7 b/res/container-tests/Containerfile.ubi7
index 9e36280..fd5b965 100644
--- a/res/container-tests/Containerfile.ubi7
+++ b/res/container-tests/Containerfile.ubi7
@@ -13,19 +13,6 @@ ENTRYPOINT virtualenv testenv && \
pip install -U funcsigs && \
pip install -U -r requirements-tests.txt && \
pip install -U . && \
- export LINTABLES=$(find . -name '*.py' | grep -E -e '^\./leapp\/' -e '^\./tests/scripts/' | sort -u ) && \
- echo '==================================================' && \
- echo '==================================================' && \
- echo '=============== Running pylint ===============' && \
- echo '==================================================' && \
- echo '==================================================' && \
- echo $LINTABLES | xargs pylint --py3k && echo '===> pylint PASSED' && \
- echo '==================================================' && \
- echo '==================================================' && \
- echo '=============== Running flake8 ===============' && \
- echo '==================================================' && \
- echo '==================================================' && \
- flake8 $LINTABLES && echo '===> flake8 PASSED' && \
echo '==================================================' && \
echo '==================================================' && \
echo '=============== Running tests ===============' && \
diff --git a/res/container-tests/Containerfile.ubi7-lint b/res/container-tests/Containerfile.ubi7-lint
new file mode 100644
index 0000000..2e1de4a
--- /dev/null
+++ b/res/container-tests/Containerfile.ubi7-lint
@@ -0,0 +1,28 @@
+FROM registry.access.redhat.com/ubi7/ubi:7.9
+
+VOLUME /payload
+
+RUN yum -y install python27-python-pip && \
+ scl enable python27 -- pip install -U --target /usr/lib/python2.7/site-packages/ pip==20.3.0 && \
+ python -m pip install --ignore-installed pip==20.3.4 virtualenv
+
+WORKDIR /payload
+ENTRYPOINT virtualenv testenv && \
+ source testenv/bin/activate && \
+ pip install -U setuptools && \
+ pip install -U funcsigs && \
+ pip install -U -r requirements-tests.txt && \
+ pip install -U . && \
+ export LINTABLES=$(find . -name '*.py' | grep -E -e '^\./leapp\/' -e '^\./tests/scripts/' | sort -u ) && \
+ echo '==================================================' && \
+ echo '==================================================' && \
+ echo '=============== Running pylint ===============' && \
+ echo '==================================================' && \
+ echo '==================================================' && \
+ echo $LINTABLES | xargs pylint --py3k && echo '===> pylint PASSED' && \
+ echo '==================================================' && \
+ echo '==================================================' && \
+ echo '=============== Running flake8 ===============' && \
+ echo '==================================================' && \
+ echo '==================================================' && \
+ flake8 $LINTABLES && echo '===> flake8 PASSED'
diff --git a/res/container-tests/Containerfile.ubi8 b/res/container-tests/Containerfile.ubi8
index 03d499d..960f4b0 100644
--- a/res/container-tests/Containerfile.ubi8
+++ b/res/container-tests/Containerfile.ubi8
@@ -16,19 +16,6 @@ ENTRYPOINT virtualenv testenv -p "/usr/bin/$PYTHON_VENV" && \
pip install -U funcsigs && \
pip install -U -r requirements-tests.txt && \
pip install -U . && \
- export LINTABLES=$(find . -name '*.py' | grep -E -e '^\./leapp\/' -e '^\./tests/scripts/' | sort -u ) && \
- echo '==================================================' && \
- echo '==================================================' && \
- echo '=============== Running pylint ===============' && \
- echo '==================================================' && \
- echo '==================================================' && \
- echo $LINTABLES | xargs pylint && echo '===> pylint PASSED' && \
- echo '==================================================' && \
- echo '==================================================' && \
- echo '=============== Running flake8 ===============' && \
- echo '==================================================' && \
- echo '==================================================' && \
- flake8 $LINTABLES && echo '===> flake8 PASSED' && \
echo '==================================================' && \
echo '==================================================' && \
echo '=============== Running tests ===============' && \
diff --git a/res/container-tests/Containerfile.ubi8-lint b/res/container-tests/Containerfile.ubi8-lint
new file mode 100644
index 0000000..75143d9
--- /dev/null
+++ b/res/container-tests/Containerfile.ubi8-lint
@@ -0,0 +1,31 @@
+FROM registry.access.redhat.com/ubi8/ubi:latest
+
+VOLUME /payload
+
+ENV PYTHON_VENV "python3.6"
+
+RUN yum update -y && \
+ yum install python3 python39 python3-virtualenv make -y && \
+ yum -y install python3-pip && \
+ python3 -m pip install --upgrade pip==20.3.4
+
+WORKDIR /payload
+ENTRYPOINT virtualenv testenv -p "/usr/bin/$PYTHON_VENV" && \
+ source testenv/bin/activate && \
+ pip install -U setuptools && \
+ pip install -U funcsigs && \
+ pip install -U -r requirements-tests.txt && \
+ pip install -U . && \
+ export LINTABLES=$(find . -name '*.py' | grep -E -e '^\./leapp\/' -e '^\./tests/scripts/' | sort -u ) && \
+ echo '==================================================' && \
+ echo '==================================================' && \
+ echo '=============== Running pylint ===============' && \
+ echo '==================================================' && \
+ echo '==================================================' && \
+ echo $LINTABLES | xargs pylint && echo '===> pylint PASSED' && \
+ echo '==================================================' && \
+ echo '==================================================' && \
+ echo '=============== Running flake8 ===============' && \
+ echo '==================================================' && \
+ echo '==================================================' && \
+ flake8 $LINTABLES && echo '===> flake8 PASSED'
diff --git a/res/container-tests/Containerfile.ubi9 b/res/container-tests/Containerfile.ubi9
index b1b5b71..755e874 100644
--- a/res/container-tests/Containerfile.ubi9
+++ b/res/container-tests/Containerfile.ubi9
@@ -15,19 +15,6 @@ ENTRYPOINT virtualenv testenv -p "/usr/bin/$PYTHON_VENV" && \
pip install -U funcsigs && \
pip install -U -r requirements-tests.txt && \
pip install -U . && \
- export LINTABLES=$(find . -name '*.py' | grep -E -e '^\./leapp\/' -e '^\./tests/scripts/' | sort -u ) && \
- echo '==================================================' && \
- echo '==================================================' && \
- echo '=============== Running pylint ===============' && \
- echo '==================================================' && \
- echo '==================================================' && \
- echo $LINTABLES | xargs pylint && echo '===> pylint PASSED' && \
- echo '==================================================' && \
- echo '==================================================' && \
- echo '=============== Running flake8 ===============' && \
- echo '==================================================' && \
- echo '==================================================' && \
- flake8 $LINTABLES && echo '===> flake8 PASSED' && \
echo '==================================================' && \
echo '==================================================' && \
echo '=============== Running tests ===============' && \
diff --git a/res/container-tests/Containerfile.ubi9-lint b/res/container-tests/Containerfile.ubi9-lint
new file mode 100644
index 0000000..9a1d186
--- /dev/null
+++ b/res/container-tests/Containerfile.ubi9-lint
@@ -0,0 +1,30 @@
+FROM registry.access.redhat.com/ubi9/ubi:latest
+
+VOLUME /payload
+
+RUN dnf update -y && \
+ dnf install python3 python39 make python3-pip -y && \
+ python3 -m pip install --upgrade pip==20.3.4
+
+RUN pip install virtualenv
+
+WORKDIR /payload
+ENTRYPOINT virtualenv testenv -p "/usr/bin/$PYTHON_VENV" && \
+ source testenv/bin/activate && \
+ pip install -U setuptools && \
+ pip install -U funcsigs && \
+ pip install -U -r requirements-tests.txt && \
+ pip install -U . && \
+ export LINTABLES=$(find . -name '*.py' | grep -E -e '^\./leapp\/' -e '^\./tests/scripts/' | sort -u ) && \
+ echo '==================================================' && \
+ echo '==================================================' && \
+ echo '=============== Running pylint ===============' && \
+ echo '==================================================' && \
+ echo '==================================================' && \
+ echo $LINTABLES | xargs pylint && echo '===> pylint PASSED' && \
+ echo '==================================================' && \
+ echo '==================================================' && \
+ echo '=============== Running flake8 ===============' && \
+ echo '==================================================' && \
+ echo '==================================================' && \
+ flake8 $LINTABLES && echo '===> flake8 PASSED'
diff --git a/res/schema/audit-layout.sql b/res/schema/audit-layout.sql
index d567ce4..93a92f1 100644
--- a/res/schema/audit-layout.sql
+++ b/res/schema/audit-layout.sql
@@ -1,6 +1,6 @@
BEGIN;
-PRAGMA user_version = 3;
+PRAGMA user_version = 4;
CREATE TABLE IF NOT EXISTS execution (
id INTEGER PRIMARY KEY NOT NULL,
@@ -47,6 +47,17 @@ CREATE TABLE IF NOT EXISTS metadata (
metadata TEXT
);
+CREATE TABLE IF NOT EXISTS actor_config (
+ id INTEGER PRIMARY KEY NOT NULL,
+ context VARCHAR(36) NOT NULL REFERENCES execution (context),
+ actor_config_hash VARCHAR(64) NOT NULL REFERENCES actor_config_data (hash)
+);
+
+CREATE TABLE IF NOT EXISTS actor_config_data (
+ hash VARCHAR(64) PRIMARY KEY NOT NULL,
+ config TEXT
+);
+
CREATE TABLE IF NOT EXISTS entity (
id INTEGER PRIMARY KEY NOT NULL,
context VARCHAR(36) NOT NULL REFERENCES execution (context),
diff --git a/res/schema/migrations/0003-add-actor-configuration.sql b/res/schema/migrations/0003-add-actor-configuration.sql
new file mode 100644
index 0000000..11131c8
--- /dev/null
+++ b/res/schema/migrations/0003-add-actor-configuration.sql
@@ -0,0 +1,16 @@
+BEGIN;
+
+CREATE TABLE IF NOT EXISTS actor_config (
+ id INTEGER PRIMARY KEY NOT NULL,
+ context VARCHAR(36) NOT NULL REFERENCES execution (context),
+ actor_config_hash VARCHAR(64) NOT NULL REFERENCES actor_config_data (hash)
+);
+
+CREATE TABLE IF NOT EXISTS actor_config_data (
+ hash VARCHAR(64) PRIMARY KEY NOT NULL,
+ config TEXT
+);
+
+PRAGMA user_version = 4;
+
+COMMIT;
diff --git a/setup.py b/setup.py
index 40e7656..3ebe8b2 100644
--- a/setup.py
+++ b/setup.py
@@ -23,7 +23,7 @@ setup(
name='leapp',
version=main_ns['VERSION'],
packages=find_packages(exclude=EXCLUSION),
- install_requires=['six', 'requests'],
+ install_requires=['six', 'requests', 'PyYAML'],
entry_points='''
[console_scripts]
snactor=leapp.snactor:main
diff --git a/tests/scripts/test_actor_config_db.py b/tests/scripts/test_actor_config_db.py
new file mode 100644
index 0000000..75d6676
--- /dev/null
+++ b/tests/scripts/test_actor_config_db.py
@@ -0,0 +1,90 @@
+import os
+import hashlib
+import json
+
+from leapp.utils.audit import (get_connection, dict_factory, ActorConfigData, ActorConfig)
+from leapp.config import get_config
+
+_CONTEXT_NAME = 'test-context-name-actor-config-db'
+_TEST_ACTOR_CONFIG = {}
+
+
+def setup_module():
+ get_config().set('database', 'path', '/tmp/leapp-test.db')
+
+
+def setup():
+ path = get_config().get('database', 'path')
+ if os.path.isfile(path):
+ os.unlink(path)
+
+
+def test_save_empty_actor_config_data():
+ hash_id = hashlib.sha256('test-empty-actor-config'.encode('utf-8')).hexdigest()
+ cfg = ActorConfigData(hash_id=hash_id, config='')
+ cfg.store()
+
+ entry = None
+ with get_connection(None) as conn:
+ cursor = conn.execute('SELECT * FROM actor_config_data WHERE hash = ?;', (hash_id,))
+ cursor.row_factory = dict_factory
+ entry = cursor.fetchone()
+
+ assert entry is not None
+ assert entry['config'] == ''
+
+
+def test_save_empty_actor_config():
+ hash_id = hashlib.sha256('test-empty-actor-config'.encode('utf-8')).hexdigest()
+ acd = ActorConfigData(hash_id=hash_id, config='')
+ ac = ActorConfig(
+ context=_CONTEXT_NAME,
+ config=acd,
+ )
+ ac.store()
+
+ assert ac.actor_config_id
+
+ entry = None
+ with get_connection(None) as conn:
+ cursor = conn.execute('SELECT * FROM actor_config WHERE id = ?;', (ac.actor_config_id,))
+ cursor.row_factory = dict_factory
+ entry = cursor.fetchone()
+
+ assert entry is not None
+ assert entry['context'] == _CONTEXT_NAME
+ assert entry['actor_config_hash'] == hash_id
+
+
+def test_store_actor_config():
+ hash_id = hashlib.sha256('test-actor-config'.encode('utf-8')).hexdigest()
+ config_str = json.dumps(_TEST_ACTOR_CONFIG, sort_keys=True)
+ acd = ActorConfigData(hash_id=hash_id, config=config_str)
+ ac = ActorConfig(
+ context=_CONTEXT_NAME,
+ config=acd,
+ )
+ ac.store()
+
+ assert ac.actor_config_id
+
+ entry = None
+ with get_connection(None) as conn:
+ cursor = conn.execute(
+ 'SELECT * FROM actor_config WHERE id = ?;', (ac.actor_config_id,))
+ cursor = conn.execute(
+ 'SELECT * '
+ 'FROM actor_config '
+ 'JOIN actor_config_data '
+ 'ON actor_config.actor_config_hash = actor_config_data.hash '
+ 'WHERE actor_config.id = ?;',
+ (ac.actor_config_id,)
+ )
+ cursor.row_factory = dict_factory
+ entry = cursor.fetchone()
+
+ assert entry is not None
+ assert entry['context'] == _CONTEXT_NAME
+ assert entry['actor_config_hash'] == hash_id
+ ans_config = json.loads(entry['config'])
+ assert ans_config == {}
diff --git a/tests/scripts/test_metadata.py b/tests/scripts/test_metadata.py
index 9b5c07f..6631c49 100644
--- a/tests/scripts/test_metadata.py
+++ b/tests/scripts/test_metadata.py
@@ -20,7 +20,8 @@ _PHASE_NAME = 'test-phase-name'
_DIALOG_SCOPE = 'test-dialog'
_WORKFLOW_METADATA_FIELDS = ('description', 'name', 'phases', 'short_name', 'tag')
-_ACTOR_METADATA_FIELDS = ('class_name', 'name', 'description', 'phase', 'tags', 'consumes', 'produces', 'path')
+_ACTOR_METADATA_FIELDS = ('class_name', 'config_schemas', 'name', 'description',
+ 'phase', 'tags', 'consumes', 'produces', 'path')
_TEST_WORKFLOW_METADATA = {
'description': 'No description has been provided for the UnitTest workflow.',
@@ -144,7 +145,7 @@ def test_store_actor_metadata(monkeypatch, repository_dir):
# ---
with repository_dir.as_cwd():
logger = logging.getLogger('leapp.actor.test')
- with mock.patch.object(logger, 'log') as log_mock:
+ with mock.patch.object(logger, 'log') as log_mock, mock.patch('os.chdir'):
definition = ActorDefinition('actors/test', '.', log=log_mock)
with mock.patch('leapp.repository.actor_definition.get_actor_metadata', return_value=_TEST_ACTOR_METADATA):
with mock.patch('leapp.repository.actor_definition.get_actors', return_value=[True]):
diff --git a/tests/scripts/test_repository_actor_definition.py b/tests/scripts/test_repository_actor_definition.py
index 3fcf757..954928f 100644
--- a/tests/scripts/test_repository_actor_definition.py
+++ b/tests/scripts/test_repository_actor_definition.py
@@ -13,6 +13,7 @@ _FAKE_META_DATA = {
'name': 'fake-actor',
'path': 'actors/test',
'tags': (),
+ 'config_schemas': (),
'consumes': (),
'produces': (),
'dialogs': (),
@@ -21,7 +22,7 @@ _FAKE_META_DATA = {
def test_actor_definition(repository_dir):
- with repository_dir.as_cwd():
+ with repository_dir.as_cwd(), mock.patch('os.chdir', return_value=None):
logger = logging.getLogger('leapp.actor.test')
with mock.patch.object(logger, 'log') as log_mock:
definition = ActorDefinition('actors/test', '.', log=log_mock)
@@ -40,6 +41,7 @@ def test_actor_definition(repository_dir):
assert definition.consumes == _FAKE_META_DATA['consumes']
assert definition.produces == _FAKE_META_DATA['produces']
assert definition.tags == _FAKE_META_DATA['tags']
+ assert definition.config_schemas == _FAKE_META_DATA['config_schemas']
assert definition.class_name == _FAKE_META_DATA['class_name']
assert definition.dialogs == _FAKE_META_DATA['dialogs']
assert definition.name == _FAKE_META_DATA['name']
@@ -52,11 +54,13 @@ def test_actor_definition(repository_dir):
assert dumped.pop('produces') == definition.produces
assert dumped.pop('apis') == definition.apis
assert dumped.pop('tags') == definition.tags
+ assert dumped.pop('config_schemas') == definition.config_schemas
assert dumped.pop('dialogs') == [dialog.serialize() for dialog in definition.dialogs]
assert dumped.pop('path') == _FAKE_META_DATA['path']
assert dumped.pop('name') == definition.name
assert dumped.pop('files') == ('.',)
assert dumped.pop('libraries') == ('.',)
+ assert dumped.pop('configs') == ('.',)
assert dumped.pop('tests') == ('.',)
assert dumped.pop('tools') == ('.',)
# Assert to ensure we covered all keys
--
2.47.0
Reference in New Issue View Git Blame Copy Permalink