1838 lines
83 KiB
Diff
1838 lines
83 KiB
Diff
|
From 1036c6b55b95a27be57b065a4b9acfecc83639b3 Mon Sep 17 00:00:00 2001
|
||
|
|
From: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
Date: Tue, 6 Jul 2021 16:02:03 -0400
|
||
|
|
Subject: [PATCH 01/19] Add tests for package_installed template
|
||
|
|
|
||
|
|
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
---
|
||
|
|
.../package_installed/tests/package-installed-removed.fail.sh | 4 ++++
|
||
|
|
.../package_installed/tests/package-installed.pass.sh | 3 +++
|
||
|
|
.../templates/package_installed/tests/package-removed.fail.sh | 3 +++
|
||
|
|
3 files changed, 10 insertions(+)
|
||
|
|
create mode 100644 shared/templates/package_installed/tests/package-installed-removed.fail.sh
|
||
|
|
create mode 100644 shared/templates/package_installed/tests/package-installed.pass.sh
|
||
|
|
create mode 100644 shared/templates/package_installed/tests/package-removed.fail.sh
|
||
|
|
|
||
|
|
diff --git a/shared/templates/package_installed/tests/package-installed-removed.fail.sh b/shared/templates/package_installed/tests/package-installed-removed.fail.sh
|
||
|
|
new file mode 100644
|
||
|
|
index 00000000000..1ce59225303
|
||
|
|
--- /dev/null
|
||
|
|
+++ b/shared/templates/package_installed/tests/package-installed-removed.fail.sh
|
||
|
|
@@ -0,0 +1,4 @@
|
||
|
|
+#!/bin/bash
|
||
|
|
+
|
||
|
|
+{{{ bash_package_install(PKGNAME) }}}
|
||
|
|
+{{{ bash_package_remove(PKGNAME) }}}
|
||
|
|
diff --git a/shared/templates/package_installed/tests/package-installed.pass.sh b/shared/templates/package_installed/tests/package-installed.pass.sh
|
||
|
|
new file mode 100644
|
||
|
|
index 00000000000..2a0506c57fd
|
||
|
|
--- /dev/null
|
||
|
|
+++ b/shared/templates/package_installed/tests/package-installed.pass.sh
|
||
|
|
@@ -0,0 +1,3 @@
|
||
|
|
+#!/bin/bash
|
||
|
|
+
|
||
|
|
+{{{ bash_package_install(PKGNAME) }}}
|
||
|
|
diff --git a/shared/templates/package_installed/tests/package-removed.fail.sh b/shared/templates/package_installed/tests/package-removed.fail.sh
|
||
|
|
new file mode 100644
|
||
|
|
index 00000000000..c2838396f56
|
||
|
|
--- /dev/null
|
||
|
|
+++ b/shared/templates/package_installed/tests/package-removed.fail.sh
|
||
|
|
@@ -0,0 +1,3 @@
|
||
|
|
+#!/bin/bash
|
||
|
|
+
|
||
|
|
+{{{ bash_package_remove(PKGNAME) }}}
|
||
|
|
|
||
|
|
From 3008339e7779b144a2b7fa72854d1185e33438f5 Mon Sep 17 00:00:00 2001
|
||
|
|
From: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
Date: Tue, 6 Jul 2021 16:16:56 -0400
|
||
|
|
Subject: [PATCH 02/19] Refactor ssg/templates.py generation (SCE)
|
||
|
|
|
||
|
|
This refactors SSG's template generation code to better align between
|
||
|
|
where we are now upstream and future changes related to templated SCE
|
||
|
|
support.
|
||
|
|
|
||
|
|
At this point in time, we wish to make some changes to allow building
|
||
|
|
templated (both in the Jinja and shared/templates senses) test cases,
|
||
|
|
and many of the SCE-enablement changes are relevant, I chose to pull
|
||
|
|
them in almost entirely. The original commit message is preserved below:
|
||
|
|
|
||
|
|
Refactor template generation for SCEs
|
||
|
|
|
||
|
|
ssg/templates.py holds the core of template generation logic. We
|
||
|
|
will need a template_builder for SCE (so we can load SCE metadata
|
||
|
|
from templated content) but we do not wish to write this SCE content
|
||
|
|
out immediately; instead, we wish to wait until
|
||
|
|
build-scripts/build_templated_content.py is run.
|
||
|
|
|
||
|
|
We refactor to make the resolved languages for a rule (the
|
||
|
|
intersection between theoretical languages allowed by the rule and
|
||
|
|
the actual languages allowed by the template) accessible to callers,
|
||
|
|
and, also allow reading just a single templated language artifact
|
||
|
|
into memory (instead of from disk).
|
||
|
|
|
||
|
|
Notably, this last change is most useful to us; we don't want to put the
|
||
|
|
file in a template/build-system specified location; we wish to control
|
||
|
|
its location exactly here.
|
||
|
|
|
||
|
|
Note that no changes to ssg/templates.py for the test suite change were
|
||
|
|
done here; this was purely a change to sync future changes for SCE
|
||
|
|
content.
|
||
|
|
|
||
|
|
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
---
|
||
|
|
ssg/templates.py | 112 +++++++++++++++++++++++++++++++++++++----------
|
||
|
|
1 file changed, 89 insertions(+), 23 deletions(-)
|
||
|
|
|
||
|
|
diff --git a/ssg/templates.py b/ssg/templates.py
|
||
|
|
index 6783d0f2d5e..47a8a5eb852 100644
|
||
|
|
--- a/ssg/templates.py
|
||
|
|
+++ b/ssg/templates.py
|
||
|
|
@@ -107,14 +107,17 @@ def __init__(
|
||
|
|
self.checks_dir = checks_dir
|
||
|
|
self.output_dirs = dict()
|
||
|
|
for lang in languages:
|
||
|
|
+ lang_dir = lang
|
||
|
|
if lang == "oval":
|
||
|
|
# OVAL checks need to be put to a different directory because
|
||
|
|
- # they are processed differently than remediations later in the
|
||
|
|
- # build process
|
||
|
|
+ # they are processed differently than remediations later in
|
||
|
|
+ # the build process
|
||
|
|
output_dir = self.checks_dir
|
||
|
|
+ if lang.startswith("sce-"):
|
||
|
|
+ lang_dir = "sce"
|
||
|
|
else:
|
||
|
|
output_dir = self.remediations_dir
|
||
|
|
- dir_ = os.path.join(output_dir, lang)
|
||
|
|
+ dir_ = os.path.join(output_dir, lang_dir)
|
||
|
|
self.output_dirs[lang] = dir_
|
||
|
|
# scan directory structure and dynamically create list of templates
|
||
|
|
for item in sorted(os.listdir(self.templates_dir)):
|
||
|
|
@@ -124,25 +127,41 @@ def __init__(
|
||
|
|
maybe_template.load()
|
||
|
|
templates[item] = maybe_template
|
||
|
|
|
||
|
|
+ def build_lang_file(
|
||
|
|
+ self, rule_id, template_name, template_vars, lang, local_env_yaml):
|
||
|
|
+ """
|
||
|
|
+ Builds and returns templated content for a given rule for a given
|
||
|
|
+ language; does not write the output to disk.
|
||
|
|
+ """
|
||
|
|
+ if lang not in templates[template_name].langs:
|
||
|
|
+ return None
|
||
|
|
+
|
||
|
|
+ template_file_name = lang + ".template"
|
||
|
|
+ template_file_path = os.path.join(self.templates_dir, template_name, template_file_name)
|
||
|
|
+ template_parameters = templates[template_name].preprocess(template_vars, lang)
|
||
|
|
+ jinja_dict = ssg.utils.merge_dicts(local_env_yaml, template_parameters)
|
||
|
|
+ filled_template = ssg.jinja.process_file_with_macros(
|
||
|
|
+ template_file_path, jinja_dict)
|
||
|
|
+
|
||
|
|
+ return filled_template
|
||
|
|
|
||
|
|
def build_lang(
|
||
|
|
- self, rule_id, template_name, template_vars, lang, local_env_yaml):
|
||
|
|
+ self, rule_id, template_name, template_vars, lang, local_env_yaml, platforms=None):
|
||
|
|
"""
|
||
|
|
Builds templated content for a given rule for a given language.
|
||
|
|
Writes the output to the correct build directories.
|
||
|
|
"""
|
||
|
|
if lang not in templates[template_name].langs:
|
||
|
|
return
|
||
|
|
- template_file_name = lang + ".template"
|
||
|
|
- template_file_path = os.path.join(self.templates_dir, template_name, template_file_name)
|
||
|
|
+
|
||
|
|
+ filled_template = self.build_lang_file(rule_id, template_name,
|
||
|
|
+ template_vars, lang, local_env_yaml)
|
||
|
|
+
|
||
|
|
ext = lang_to_ext_map[lang]
|
||
|
|
output_file_name = rule_id + ext
|
||
|
|
output_filepath = os.path.join(
|
||
|
|
self.output_dirs[lang], output_file_name)
|
||
|
|
- template_parameters = templates[template_name].preprocess(template_vars, lang)
|
||
|
|
- jinja_dict = ssg.utils.merge_dicts(local_env_yaml, template_parameters)
|
||
|
|
- filled_template = ssg.jinja.process_file_with_macros(
|
||
|
|
- template_file_path, jinja_dict)
|
||
|
|
+
|
||
|
|
with open(output_filepath, "w") as f:
|
||
|
|
f.write(filled_template)
|
||
|
|
|
||
|
|
@@ -168,6 +187,37 @@ def get_langs_to_generate(self, rule):
|
||
|
|
else:
|
||
|
|
return languages
|
||
|
|
|
||
|
|
+ def get_template_name(self, template):
|
||
|
|
+ """
|
||
|
|
+ Given a template dictionary from a Rule instance, determine the name
|
||
|
|
+ of the template (from templates) this rule uses.
|
||
|
|
+ """
|
||
|
|
+ try:
|
||
|
|
+ template_name = template["name"]
|
||
|
|
+ except KeyError:
|
||
|
|
+ raise ValueError(
|
||
|
|
+ "Rule {0} is missing template name under template key".format(
|
||
|
|
+ rule_id))
|
||
|
|
+ if template_name not in templates.keys():
|
||
|
|
+ raise ValueError(
|
||
|
|
+ "Rule {0} uses template {1} which does not exist.".format(
|
||
|
|
+ rule_id, template_name))
|
||
|
|
+ return template_name
|
||
|
|
+
|
||
|
|
+ def get_resolved_langs_to_generate(self, rule):
|
||
|
|
+ """
|
||
|
|
+ Given a specific Rule instance, determine which languages are
|
||
|
|
+ generated by the combination of the rule's template_backends AND
|
||
|
|
+ the rule's template keys.
|
||
|
|
+ """
|
||
|
|
+ if rule.template is None:
|
||
|
|
+ return None
|
||
|
|
+
|
||
|
|
+ rule_langs = set(self.get_langs_to_generate(rule))
|
||
|
|
+ template_name = self.get_template_name(rule.template)
|
||
|
|
+ template_langs = set(templates[template_name].langs)
|
||
|
|
+ return rule_langs.intersection(template_langs)
|
||
|
|
+
|
||
|
|
def process_product_vars(self, all_variables):
|
||
|
|
"""
|
||
|
|
Given a dictionary with the format key[@<product>]=value, filter out
|
||
|
|
@@ -183,21 +233,12 @@ def process_product_vars(self, all_variables):
|
||
|
|
|
||
|
|
return processed
|
||
|
|
|
||
|
|
- def build_rule(self, rule_id, rule_title, template, langs_to_generate):
|
||
|
|
+ def build_rule(self, rule_id, rule_title, template, langs_to_generate, platforms=None):
|
||
|
|
"""
|
||
|
|
Builds templated content for a given rule for selected languages,
|
||
|
|
writing the output to the correct build directories.
|
||
|
|
"""
|
||
|
|
- try:
|
||
|
|
- template_name = template["name"]
|
||
|
|
- except KeyError:
|
||
|
|
- raise ValueError(
|
||
|
|
- "Rule {0} is missing template name under template key".format(
|
||
|
|
- rule_id))
|
||
|
|
- if template_name not in templates.keys():
|
||
|
|
- raise ValueError(
|
||
|
|
- "Rule {0} uses template {1} which does not exist.".format(
|
||
|
|
- rule_id, template_name))
|
||
|
|
+ template_name = self.get_template_name(template)
|
||
|
|
try:
|
||
|
|
template_vars = self.process_product_vars(template["vars"])
|
||
|
|
except KeyError:
|
||
|
|
@@ -216,11 +257,36 @@ def build_rule(self, rule_id, rule_title, template, langs_to_generate):
|
||
|
|
for lang in langs_to_generate:
|
||
|
|
try:
|
||
|
|
self.build_lang(
|
||
|
|
- rule_id, template_name, template_vars, lang, local_env_yaml)
|
||
|
|
+ rule_id, template_name, template_vars, lang, local_env_yaml, platforms)
|
||
|
|
except Exception as e:
|
||
|
|
print("Error building templated {0} content for rule {1}".format(lang, rule_id), file=sys.stderr)
|
||
|
|
raise e
|
||
|
|
|
||
|
|
+ def get_lang_for_rule(self, rule_id, rule_title, template, language):
|
||
|
|
+ """
|
||
|
|
+ For the specified rule, build and return only the specified language
|
||
|
|
+ content.
|
||
|
|
+ """
|
||
|
|
+ template_name = self.get_template_name(template)
|
||
|
|
+ try:
|
||
|
|
+ template_vars = self.process_product_vars(template["vars"])
|
||
|
|
+ except KeyError:
|
||
|
|
+ raise ValueError(
|
||
|
|
+ "Rule {0} does not contain mandatory 'vars:' key under "
|
||
|
|
+ "'template:' key.".format(rule_id))
|
||
|
|
+ # Add the rule ID which will be reused in OVAL templates as OVAL
|
||
|
|
+ # definition ID so that the build system matches the generated
|
||
|
|
+ # check with the rule.
|
||
|
|
+ template_vars["_rule_id"] = rule_id
|
||
|
|
+ # checks and remediations are processed with a custom YAML dict
|
||
|
|
+ local_env_yaml = self.env_yaml.copy()
|
||
|
|
+ local_env_yaml["rule_id"] = rule_id
|
||
|
|
+ local_env_yaml["rule_title"] = rule_title
|
||
|
|
+ local_env_yaml["products"] = self.env_yaml["product"]
|
||
|
|
+
|
||
|
|
+ return self.build_lang_file(rule_id, template_name, template_vars,
|
||
|
|
+ language, local_env_yaml)
|
||
|
|
+
|
||
|
|
def build_extra_ovals(self):
|
||
|
|
declaration_path = os.path.join(self.templates_dir, "extra_ovals.yml")
|
||
|
|
declaration = ssg.yaml.open_raw(declaration_path)
|
||
|
|
@@ -245,7 +311,7 @@ def build_all_rules(self):
|
||
|
|
continue
|
||
|
|
langs_to_generate = self.get_langs_to_generate(rule)
|
||
|
|
self.build_rule(
|
||
|
|
- rule.id_, rule.title, rule.template, langs_to_generate)
|
||
|
|
+ rule.id_, rule.title, rule.template, langs_to_generate, platforms=rule.platforms)
|
||
|
|
|
||
|
|
def build(self):
|
||
|
|
"""
|
||
|
|
|
||
|
|
From 8da04c6a85d40432a19904b16251885e12b75999 Mon Sep 17 00:00:00 2001
|
||
|
|
From: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
Date: Tue, 6 Jul 2021 16:50:06 -0400
|
||
|
|
Subject: [PATCH 03/19] Remove incorrect tests for package installation
|
||
|
|
|
||
|
|
These tests assume that the package manager is yum (and forcibly
|
||
|
|
installs it) in some cases. This doesn't work on Debian-like systems;
|
||
|
|
prefer the templated version instead, which uses the
|
||
|
|
bash_package_install Jinja macro instead (and uses the
|
||
|
|
bash_package_remove macro for the negative case)
|
||
|
|
|
||
|
|
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
---
|
||
|
|
.../ntp/package_chrony_installed/tests/installed.pass.sh | 4 ----
|
||
|
|
.../ntp/package_chrony_installed/tests/removed.fail.sh | 3 ---
|
||
|
|
.../package_audit_installed/tests/installed.pass.sh | 4 ----
|
||
|
|
.../package_audit_installed/tests/removed.fail.sh | 3 ---
|
||
|
|
.../package_rsyslog_installed/tests/installed.pass.sh | 3 ---
|
||
|
|
.../package_rsyslog_installed/tests/notinstalled.fail.sh | 3 ---
|
||
|
|
.../package_firewalld_installed/tests/installed.pass.sh | 4 ----
|
||
|
|
.../package_firewalld_installed/tests/removed.fail.sh | 3 ---
|
||
|
|
.../package_libselinux_installed/tests/installed.pass.sh | 4 ----
|
||
|
|
.../aide/package_aide_installed/tests/installed.pass.sh | 3 ---
|
||
|
|
.../aide/package_aide_installed/tests/notinstalled.fail.sh | 3 ---
|
||
|
|
.../sudo/package_sudo_installed/tests/installed.pass.sh | 4 ----
|
||
|
|
.../sudo/package_sudo_installed/tests/removed.fail.sh | 3 ---
|
||
|
|
13 files changed, 53 deletions(-)
|
||
|
|
delete mode 100644 linux_os/guide/services/ntp/package_chrony_installed/tests/installed.pass.sh
|
||
|
|
delete mode 100644 linux_os/guide/services/ntp/package_chrony_installed/tests/removed.fail.sh
|
||
|
|
delete mode 100644 linux_os/guide/system/auditing/package_audit_installed/tests/installed.pass.sh
|
||
|
|
delete mode 100644 linux_os/guide/system/auditing/package_audit_installed/tests/removed.fail.sh
|
||
|
|
delete mode 100644 linux_os/guide/system/logging/package_rsyslog_installed/tests/installed.pass.sh
|
||
|
|
delete mode 100644 linux_os/guide/system/logging/package_rsyslog_installed/tests/notinstalled.fail.sh
|
||
|
|
delete mode 100644 linux_os/guide/system/network/network-firewalld/firewalld_activation/package_firewalld_installed/tests/installed.pass.sh
|
||
|
|
delete mode 100644 linux_os/guide/system/network/network-firewalld/firewalld_activation/package_firewalld_installed/tests/removed.fail.sh
|
||
|
|
delete mode 100644 linux_os/guide/system/selinux/package_libselinux_installed/tests/installed.pass.sh
|
||
|
|
delete mode 100644 linux_os/guide/system/software/integrity/software-integrity/aide/package_aide_installed/tests/installed.pass.sh
|
||
|
|
delete mode 100644 linux_os/guide/system/software/integrity/software-integrity/aide/package_aide_installed/tests/notinstalled.fail.sh
|
||
|
|
delete mode 100644 linux_os/guide/system/software/sudo/package_sudo_installed/tests/installed.pass.sh
|
||
|
|
delete mode 100644 linux_os/guide/system/software/sudo/package_sudo_installed/tests/removed.fail.sh
|
||
|
|
|
||
|
|
diff --git a/linux_os/guide/services/ntp/package_chrony_installed/tests/installed.pass.sh b/linux_os/guide/services/ntp/package_chrony_installed/tests/installed.pass.sh
|
||
|
|
deleted file mode 100644
|
||
|
|
index c692df2b282..00000000000
|
||
|
|
--- a/linux_os/guide/services/ntp/package_chrony_installed/tests/installed.pass.sh
|
||
|
|
+++ /dev/null
|
||
|
|
@@ -1,4 +0,0 @@
|
||
|
|
-#!/bin/bash
|
||
|
|
-# package = yum
|
||
|
|
-
|
||
|
|
-yum install -y chrony
|
||
|
|
diff --git a/linux_os/guide/services/ntp/package_chrony_installed/tests/removed.fail.sh b/linux_os/guide/services/ntp/package_chrony_installed/tests/removed.fail.sh
|
||
|
|
deleted file mode 100644
|
||
|
|
index eba551f99d4..00000000000
|
||
|
|
--- a/linux_os/guide/services/ntp/package_chrony_installed/tests/removed.fail.sh
|
||
|
|
+++ /dev/null
|
||
|
|
@@ -1,3 +0,0 @@
|
||
|
|
-#!/bin/bash
|
||
|
|
-
|
||
|
|
-yum remove -y chrony
|
||
|
|
diff --git a/linux_os/guide/system/auditing/package_audit_installed/tests/installed.pass.sh b/linux_os/guide/system/auditing/package_audit_installed/tests/installed.pass.sh
|
||
|
|
deleted file mode 100644
|
||
|
|
index 1c19ce31a87..00000000000
|
||
|
|
--- a/linux_os/guide/system/auditing/package_audit_installed/tests/installed.pass.sh
|
||
|
|
+++ /dev/null
|
||
|
|
@@ -1,4 +0,0 @@
|
||
|
|
-#!/bin/bash
|
||
|
|
-# package = yum
|
||
|
|
-
|
||
|
|
-yum install -y audit
|
||
|
|
diff --git a/linux_os/guide/system/auditing/package_audit_installed/tests/removed.fail.sh b/linux_os/guide/system/auditing/package_audit_installed/tests/removed.fail.sh
|
||
|
|
deleted file mode 100644
|
||
|
|
index 45dd05a9638..00000000000
|
||
|
|
--- a/linux_os/guide/system/auditing/package_audit_installed/tests/removed.fail.sh
|
||
|
|
+++ /dev/null
|
||
|
|
@@ -1,3 +0,0 @@
|
||
|
|
-#!/bin/bash
|
||
|
|
-
|
||
|
|
-yum remove -y audit
|
||
|
|
diff --git a/linux_os/guide/system/logging/package_rsyslog_installed/tests/installed.pass.sh b/linux_os/guide/system/logging/package_rsyslog_installed/tests/installed.pass.sh
|
||
|
|
deleted file mode 100644
|
||
|
|
index 6e4107fa3bd..00000000000
|
||
|
|
--- a/linux_os/guide/system/logging/package_rsyslog_installed/tests/installed.pass.sh
|
||
|
|
+++ /dev/null
|
||
|
|
@@ -1,3 +0,0 @@
|
||
|
|
-#!/bin/bash
|
||
|
|
-# packages = rsyslog
|
||
|
|
-
|
||
|
|
diff --git a/linux_os/guide/system/logging/package_rsyslog_installed/tests/notinstalled.fail.sh b/linux_os/guide/system/logging/package_rsyslog_installed/tests/notinstalled.fail.sh
|
||
|
|
deleted file mode 100644
|
||
|
|
index f64bf1b340a..00000000000
|
||
|
|
--- a/linux_os/guide/system/logging/package_rsyslog_installed/tests/notinstalled.fail.sh
|
||
|
|
+++ /dev/null
|
||
|
|
@@ -1,3 +0,0 @@
|
||
|
|
-#!/bin/bash
|
||
|
|
-
|
||
|
|
-yum remove -y rsyslog
|
||
|
|
diff --git a/linux_os/guide/system/network/network-firewalld/firewalld_activation/package_firewalld_installed/tests/installed.pass.sh b/linux_os/guide/system/network/network-firewalld/firewalld_activation/package_firewalld_installed/tests/installed.pass.sh
|
||
|
|
deleted file mode 100644
|
||
|
|
index 7a4748863bf..00000000000
|
||
|
|
--- a/linux_os/guide/system/network/network-firewalld/firewalld_activation/package_firewalld_installed/tests/installed.pass.sh
|
||
|
|
+++ /dev/null
|
||
|
|
@@ -1,4 +0,0 @@
|
||
|
|
-#!/bin/bash
|
||
|
|
-# package = yum
|
||
|
|
-
|
||
|
|
-yum install -y firewalld
|
||
|
|
diff --git a/linux_os/guide/system/network/network-firewalld/firewalld_activation/package_firewalld_installed/tests/removed.fail.sh b/linux_os/guide/system/network/network-firewalld/firewalld_activation/package_firewalld_installed/tests/removed.fail.sh
|
||
|
|
deleted file mode 100644
|
||
|
|
index a9107df7de8..00000000000
|
||
|
|
--- a/linux_os/guide/system/network/network-firewalld/firewalld_activation/package_firewalld_installed/tests/removed.fail.sh
|
||
|
|
+++ /dev/null
|
||
|
|
@@ -1,3 +0,0 @@
|
||
|
|
-#!/bin/bash
|
||
|
|
-
|
||
|
|
-yum remove -y firewalld
|
||
|
|
diff --git a/linux_os/guide/system/selinux/package_libselinux_installed/tests/installed.pass.sh b/linux_os/guide/system/selinux/package_libselinux_installed/tests/installed.pass.sh
|
||
|
|
deleted file mode 100644
|
||
|
|
index 5d30cf77841..00000000000
|
||
|
|
--- a/linux_os/guide/system/selinux/package_libselinux_installed/tests/installed.pass.sh
|
||
|
|
+++ /dev/null
|
||
|
|
@@ -1,4 +0,0 @@
|
||
|
|
-#!/bin/bash
|
||
|
|
-# package = yum
|
||
|
|
-
|
||
|
|
-yum install -y libselinux
|
||
|
|
diff --git a/linux_os/guide/system/software/integrity/software-integrity/aide/package_aide_installed/tests/installed.pass.sh b/linux_os/guide/system/software/integrity/software-integrity/aide/package_aide_installed/tests/installed.pass.sh
|
||
|
|
deleted file mode 100644
|
||
|
|
index fa8b85b..0000000
|
||
|
|
--- a/linux_os/guide/system/software/integrity/software-integrity/aide/package_aide_installed/tests/installed.pass.sh
|
||
|
|
+++ /dev/null
|
||
|
|
@@ -1,3 +0,0 @@
|
||
|
|
-#!/bin/bash
|
||
|
|
-# packages = aide
|
||
|
|
-
|
||
|
|
diff --git a/linux_os/guide/system/software/integrity/software-integrity/aide/package_aide_installed/tests/notinstalled.fail.sh b/linux_os/guide/system/software/integrity/software-integrity/aide/package_aide_installed/tests/notinstalled.fail.sh
|
||
|
|
deleted file mode 100644
|
||
|
|
index 75978b6..0000000
|
||
|
|
--- a/linux_os/guide/system/software/integrity/software-integrity/aide/package_aide_installed/tests/notinstalled.fail.sh
|
||
|
|
+++ /dev/null
|
||
|
|
@@ -1,3 +0,0 @@
|
||
|
|
-#!/bin/bash
|
||
|
|
-
|
||
|
|
-yum remove -y aide
|
||
|
|
diff --git a/linux_os/guide/system/software/sudo/package_sudo_installed/tests/installed.pass.sh b/linux_os/guide/system/software/sudo/package_sudo_installed/tests/installed.pass.sh
|
||
|
|
deleted file mode 100644
|
||
|
|
index dafc6998a9a..00000000000
|
||
|
|
--- a/linux_os/guide/system/software/sudo/package_sudo_installed/tests/installed.pass.sh
|
||
|
|
+++ /dev/null
|
||
|
|
@@ -1,4 +0,0 @@
|
||
|
|
-#!/bin/bash
|
||
|
|
-# package = yum
|
||
|
|
-
|
||
|
|
-yum install -y sudo
|
||
|
|
diff --git a/linux_os/guide/system/software/sudo/package_sudo_installed/tests/removed.fail.sh b/linux_os/guide/system/software/sudo/package_sudo_installed/tests/removed.fail.sh
|
||
|
|
deleted file mode 100644
|
||
|
|
index d22b562c15e..00000000000
|
||
|
|
--- a/linux_os/guide/system/software/sudo/package_sudo_installed/tests/removed.fail.sh
|
||
|
|
+++ /dev/null
|
||
|
|
@@ -1,3 +0,0 @@
|
||
|
|
-#!/bin/bash
|
||
|
|
-
|
||
|
|
-rpm -e --nodeps sudo
|
||
|
|
|
||
|
|
From 6c49dcb79aaf5b239f09ecfdac07b443736b5b5e Mon Sep 17 00:00:00 2001
|
||
|
|
From: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
Date: Tue, 6 Jul 2021 16:51:05 -0400
|
||
|
|
Subject: [PATCH 04/19] Support templating test content
|
||
|
|
|
||
|
|
We introduced a new method to the template builder in ssg/templates.py
|
||
|
|
for finding (and building) test content under shared/templates/...
|
||
|
|
This uses Jinja macros and has the full context of the template present,
|
||
|
|
so for instance, package_installed tests can use the correct package
|
||
|
|
name on the target platform.
|
||
|
|
|
||
|
|
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
---
|
||
|
|
ssg/templates.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++
|
||
|
|
1 file changed, 46 insertions(+)
|
||
|
|
|
||
|
|
diff --git a/ssg/templates.py b/ssg/templates.py
|
||
|
|
index 47a8a5eb852..0a39c3ba094 100644
|
||
|
|
--- a/ssg/templates.py
|
||
|
|
+++ b/ssg/templates.py
|
||
|
|
@@ -145,6 +145,52 @@ def build_lang_file(
|
||
|
|
|
||
|
|
return filled_template
|
||
|
|
|
||
|
|
+ def get_all_tests(
|
||
|
|
+ self, rule_id, rule_template, local_env_yaml, platforms=None):
|
||
|
|
+ """
|
||
|
|
+ Builds a dictionary of a test case path -> test case value mapping.
|
||
|
|
+
|
||
|
|
+ Here, we want to know what the relative path on disk (under the tests/
|
||
|
|
+ subdirectory) is (such as "installed.pass.sh"), along with the actual
|
||
|
|
+ contents of the test case.
|
||
|
|
+
|
||
|
|
+ Presumably, we'll find the test case we want (all of them when
|
||
|
|
+ building a test case tarball) and write them to disk in the
|
||
|
|
+ appropriate location.
|
||
|
|
+ """
|
||
|
|
+ template_name = rule_template['name']
|
||
|
|
+ template_vars = rule_template['vars']
|
||
|
|
+
|
||
|
|
+ base_dir = os.path.abspath(os.path.join(self.templates_dir, template_name, "tests"))
|
||
|
|
+ results = dict()
|
||
|
|
+
|
||
|
|
+ # If no test cases exist, return an empty dictionary.
|
||
|
|
+ if not os.path.exists(base_dir):
|
||
|
|
+ return results
|
||
|
|
+
|
||
|
|
+ # Walk files; note that we don't need to do anything about directories
|
||
|
|
+ # as only files are recorded in the mapping; directories can be
|
||
|
|
+ # inferred from the path.
|
||
|
|
+ for dirpath, _, filenames in os.walk(base_dir):
|
||
|
|
+ if not filenames:
|
||
|
|
+ continue
|
||
|
|
+
|
||
|
|
+ for filename in filenames:
|
||
|
|
+ # Relative path to the file becomes our results key.
|
||
|
|
+ absolute_path = os.path.abspath(os.path.join(dirpath, filename))
|
||
|
|
+ relative_path = os.path.relpath(absolute_path, base_dir)
|
||
|
|
+
|
||
|
|
+ # Load template parameters and apply it to the test case.
|
||
|
|
+ template_parameters = templates[template_name].preprocess(template_vars, "tests")
|
||
|
|
+ jinja_dict = ssg.utils.merge_dicts(local_env_yaml, template_parameters)
|
||
|
|
+ filled_template = ssg.jinja.process_file_with_macros(
|
||
|
|
+ absolute_path, jinja_dict)
|
||
|
|
+
|
||
|
|
+ # Save the results under the relative path.
|
||
|
|
+ results[relative_path] = filled_template
|
||
|
|
+
|
||
|
|
+ return results
|
||
|
|
+
|
||
|
|
def build_lang(
|
||
|
|
self, rule_id, template_name, template_vars, lang, local_env_yaml, platforms=None):
|
||
|
|
"""
|
||
|
|
|
||
|
|
From 4bd5061ffbb524b09975982fbd6dae26f35770c3 Mon Sep 17 00:00:00 2001
|
||
|
|
From: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
Date: Tue, 6 Jul 2021 16:53:44 -0400
|
||
|
|
Subject: [PATCH 05/19] Add templated tests into templated directory
|
||
|
|
|
||
|
|
This extends the existing templated directory generation to also include
|
||
|
|
tests under shared/templates/.../tests. The complicated portion is in
|
||
|
|
subdirectory handling: we need to ensure we create all necessary nested
|
||
|
|
subdirectories that are implicitly implied by the relative paths. This
|
||
|
|
is a shortcoming in the get_all_tests's return format.
|
||
|
|
|
||
|
|
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
---
|
||
|
|
tests/ssg_test_suite/common.py | 43 ++++++++++++++++++++++++++++++++++
|
||
|
|
1 file changed, 43 insertions(+)
|
||
|
|
|
||
|
|
diff --git a/tests/ssg_test_suite/common.py b/tests/ssg_test_suite/common.py
|
||
|
|
index 3dbeaf304a4..2bd4b2bd1c7 100644
|
||
|
|
--- a/tests/ssg_test_suite/common.py
|
||
|
|
+++ b/tests/ssg_test_suite/common.py
|
||
|
|
@@ -21,6 +21,9 @@
|
||
|
|
from ssg.rule_yaml import parse_prodtype
|
||
|
|
from ssg_test_suite.log import LogHelper
|
||
|
|
|
||
|
|
+import ssg.templates
|
||
|
|
+
|
||
|
|
+
|
||
|
|
Scenario_run = namedtuple(
|
||
|
|
"Scenario_run",
|
||
|
|
("rule_id", "script"))
|
||
|
|
@@ -39,6 +42,8 @@
|
||
|
|
|
||
|
|
_SHARED_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '../shared'))
|
||
|
|
|
||
|
|
+_SHARED_TEMPLATES = os.path.abspath(os.path.join(SSG_ROOT, 'shared/templates'))
|
||
|
|
+
|
||
|
|
REMOTE_USER = "root"
|
||
|
|
REMOTE_USER_HOME_DIRECTORY = "/root"
|
||
|
|
REMOTE_TEST_SCENARIOS_DIRECTORY = os.path.join(REMOTE_USER_HOME_DIRECTORY, "ssgts")
|
||
|
|
@@ -278,6 +283,11 @@ def template_tests(product=None):
|
||
|
|
yaml_path = product_yaml_path(SSG_ROOT, product)
|
||
|
|
product_yaml = load_product_yaml(yaml_path)
|
||
|
|
|
||
|
|
+ # Initialize a mock template_builder.
|
||
|
|
+ empty = "/ssgts/empty/placeholder"
|
||
|
|
+ template_builder = ssg.templates.Builder(product_yaml, empty,
|
||
|
|
+ _SHARED_TEMPLATES, empty, empty)
|
||
|
|
+
|
||
|
|
# Below we could run into a DocumentationNotComplete error. However,
|
||
|
|
# because the test suite isn't executed in the context of a particular
|
||
|
|
# build (though, ideally it would be linked), we may not know exactly
|
||
|
|
@@ -299,8 +309,11 @@ def template_tests(product=None):
|
||
|
|
|
||
|
|
# Load rule content in our environment. We use this to satisfy
|
||
|
|
# some implied properties that might be used in the test suite.
|
||
|
|
+ # Make sure we normalize to a specific product as well so that
|
||
|
|
+ # when we load templated content it is correct.
|
||
|
|
rule_path = get_rule_dir_yaml(dirpath)
|
||
|
|
rule = RuleYAML.from_yaml(rule_path, product_yaml)
|
||
|
|
+ rule.normalize(product)
|
||
|
|
|
||
|
|
# Note that most places would check prodtype, but we don't care
|
||
|
|
# about that here: if the rule is available to the product, we
|
||
|
|
@@ -320,6 +333,36 @@ def template_tests(product=None):
|
||
|
|
dest_path = os.path.join(tmpdir, rule.id_)
|
||
|
|
os.mkdir(dest_path)
|
||
|
|
|
||
|
|
+ # The priority order is rule-specific tests over templated tests.
|
||
|
|
+ # That is, for any test under rule_id/tests with a name matching a
|
||
|
|
+ # test under shared/templates/<template_name>/tests/, the former
|
||
|
|
+ # will preferred. This means we need to process templates first,
|
||
|
|
+ # so they'll be overwritten later if necessary.
|
||
|
|
+ if rule.template:
|
||
|
|
+ templated_tests = template_builder.get_all_tests(
|
||
|
|
+ rule.id_, rule.template, local_env_yaml)
|
||
|
|
+
|
||
|
|
+ for relative_path in templated_tests:
|
||
|
|
+ output_path = os.path.join(dest_path, relative_path)
|
||
|
|
+
|
||
|
|
+ # If there's a separator in the file name, it means we
|
||
|
|
+ # have nested directories to deal with.
|
||
|
|
+ if os.path.sep in relative_path:
|
||
|
|
+ parts = os.path.split(relative_path)[:-1]
|
||
|
|
+ for subdir_index in range(len(parts)):
|
||
|
|
+ # We need to expand all directories in correct
|
||
|
|
+ # order, preserving any previous directories (as
|
||
|
|
+ # they're nested). Use the star operator to splat
|
||
|
|
+ # array parts into arguments to os.path.join(...).
|
||
|
|
+ new_directory = os.path.join(dest_path, *parts[:subdir_index])
|
||
|
|
+ os.mkdir(new_directory)
|
||
|
|
+
|
||
|
|
+ # Write out the test content to the desired location on
|
||
|
|
+ # disk.
|
||
|
|
+ with open(output_path, 'w') as output_fp:
|
||
|
|
+ test_content = templated_tests[relative_path]
|
||
|
|
+ print(test_content, file=output_fp)
|
||
|
|
+
|
||
|
|
# Walk the test directory, writing all tests into the output
|
||
|
|
# directory, recursively.
|
||
|
|
tests_dir_path = os.path.join(dirpath, "tests")
|
||
|
|
|
||
|
|
From 013b247c1be897bd12ae4cf2bc2443279b1a52c9 Mon Sep 17 00:00:00 2001
|
||
|
|
From: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
Date: Wed, 7 Jul 2021 07:00:37 -0400
|
||
|
|
Subject: [PATCH 06/19] Teach iterate_over_rules about templated tests
|
||
|
|
|
||
|
|
There are two parts to rule-based test execution:
|
||
|
|
|
||
|
|
1. Building the tarball of tests and shipping it over to the remote
|
||
|
|
machine.
|
||
|
|
2. Iterating over these tests to see which scenarios are actually
|
||
|
|
applicable and any other metadata actions we need to take.
|
||
|
|
|
||
|
|
In particular, item two is controlled by iterate_over_rules. Here we
|
||
|
|
need to:
|
||
|
|
|
||
|
|
1. Teach it about the templating system.
|
||
|
|
2. Switch it from returning a list of test cases to a dictionary
|
||
|
|
mapping test case -> contents. This allows us to contain the
|
||
|
|
templating system to common.py only and not leak it into other parts
|
||
|
|
of the test suite.
|
||
|
|
3. Do a little bit of refactoring to contain shared code between
|
||
|
|
iterate_over_rules and template_tests.
|
||
|
|
|
||
|
|
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
---
|
||
|
|
tests/ssg_test_suite/common.py | 146 ++++++++++++++++++++++++---------
|
||
|
|
1 file changed, 108 insertions(+), 38 deletions(-)
|
||
|
|
|
||
|
|
diff --git a/tests/ssg_test_suite/common.py b/tests/ssg_test_suite/common.py
|
||
|
|
index 2bd4b2bd1c7..b357f816027 100644
|
||
|
|
--- a/tests/ssg_test_suite/common.py
|
||
|
|
+++ b/tests/ssg_test_suite/common.py
|
||
|
|
@@ -264,6 +264,62 @@ def _rel_abs_path(current_path, base_path):
|
||
|
|
return os.path.relpath(current_path, base_path)
|
||
|
|
|
||
|
|
|
||
|
|
+def get_product_context(product=None):
|
||
|
|
+ """
|
||
|
|
+ Returns a product YAML context if any product is specified. Hard-coded to
|
||
|
|
+ assume a debug build.
|
||
|
|
+ """
|
||
|
|
+ # Load product's YAML file if present. This will allow us to parse
|
||
|
|
+ # tests in the context of the product we're executing under.
|
||
|
|
+ product_yaml = dict()
|
||
|
|
+ if product:
|
||
|
|
+ yaml_path = product_yaml_path(SSG_ROOT, product)
|
||
|
|
+ product_yaml = load_product_yaml(yaml_path)
|
||
|
|
+
|
||
|
|
+ # We could run into a DocumentationNotComplete error when loading a
|
||
|
|
+ # rule's YAML contents. However, because the test suite isn't executed
|
||
|
|
+ # in the context of a particular build (though, ideally it would be
|
||
|
|
+ # linked), we may not know exactly whether the top-level rule/profile
|
||
|
|
+ # we're testing is actually completed. Thus, forcibly set the required
|
||
|
|
+ # property to bypass this error.
|
||
|
|
+ product_yaml['cmake_build_type'] = 'Debug'
|
||
|
|
+
|
||
|
|
+ return product_yaml
|
||
|
|
+
|
||
|
|
+
|
||
|
|
+def load_rule_and_env(rule_dir_path, env_yaml, product=None):
|
||
|
|
+ """
|
||
|
|
+ Loads a rule and returns the combination of the RuleYAML class and
|
||
|
|
+ the corresponding local environment for that rule.
|
||
|
|
+ """
|
||
|
|
+
|
||
|
|
+ # First build the path to the rule.yml file
|
||
|
|
+ rule_path = get_rule_dir_yaml(rule_dir_path)
|
||
|
|
+
|
||
|
|
+ # Load rule content in our environment. We use this to satisfy
|
||
|
|
+ # some implied properties that might be used in the test suite.
|
||
|
|
+ # Make sure we normalize to a specific product as well so that
|
||
|
|
+ # when we load templated content it is correct.
|
||
|
|
+ rule = RuleYAML.from_yaml(rule_path, env_yaml)
|
||
|
|
+ rule.normalize(product)
|
||
|
|
+
|
||
|
|
+ # Note that most places would check prodtype, but we don't care
|
||
|
|
+ # about that here: if the rule is available to the product, we
|
||
|
|
+ # load and parse it anyways as we have no knowledge of the
|
||
|
|
+ # top-level profile or rule passed into the test suite.
|
||
|
|
+ prodtypes = parse_prodtype(rule.prodtype)
|
||
|
|
+
|
||
|
|
+ # Our local copy of env_yaml needs some properties from rule.yml
|
||
|
|
+ # for completeness.
|
||
|
|
+ local_env_yaml = dict()
|
||
|
|
+ local_env_yaml.update(env_yaml)
|
||
|
|
+ local_env_yaml['rule_id'] = rule.id_
|
||
|
|
+ local_env_yaml['rule_title'] = rule.title
|
||
|
|
+ local_env_yaml['products'] = prodtypes
|
||
|
|
+
|
||
|
|
+ return rule, local_env_yaml
|
||
|
|
+
|
||
|
|
+
|
||
|
|
def template_tests(product=None):
|
||
|
|
"""
|
||
|
|
Create a temporary directory with test cases parsed via jinja using
|
||
|
|
@@ -276,26 +332,14 @@ def template_tests(product=None):
|
||
|
|
# it on success. Wrap in a try/except block and reraise the original
|
||
|
|
# exception after removing the temporary directory.
|
||
|
|
try:
|
||
|
|
- # Load product's YAML file if present. This will allow us to parse
|
||
|
|
- # tests in the context of the product we're executing under.
|
||
|
|
- product_yaml = dict()
|
||
|
|
- if product:
|
||
|
|
- yaml_path = product_yaml_path(SSG_ROOT, product)
|
||
|
|
- product_yaml = load_product_yaml(yaml_path)
|
||
|
|
+ # Load the product context we're executing under, if any.
|
||
|
|
+ product_yaml = get_product_context(product)
|
||
|
|
|
||
|
|
# Initialize a mock template_builder.
|
||
|
|
empty = "/ssgts/empty/placeholder"
|
||
|
|
template_builder = ssg.templates.Builder(product_yaml, empty,
|
||
|
|
_SHARED_TEMPLATES, empty, empty)
|
||
|
|
|
||
|
|
- # Below we could run into a DocumentationNotComplete error. However,
|
||
|
|
- # because the test suite isn't executed in the context of a particular
|
||
|
|
- # build (though, ideally it would be linked), we may not know exactly
|
||
|
|
- # whether the top-level rule/profile we're testing is actually
|
||
|
|
- # completed. Thus, forcibly set the required property to bypass this
|
||
|
|
- # error.
|
||
|
|
- product_yaml['cmake_build_type'] = 'Debug'
|
||
|
|
-
|
||
|
|
# Note that we're not exactly copying 1-for-1 the contents of the
|
||
|
|
# directory structure into the temporary one. Instead we want a
|
||
|
|
# flattened mapping with all rules in a single top-level directory
|
||
|
|
@@ -307,27 +351,8 @@ def template_tests(product=None):
|
||
|
|
if "tests" not in dirnames or not is_rule_dir(dirpath):
|
||
|
|
continue
|
||
|
|
|
||
|
|
- # Load rule content in our environment. We use this to satisfy
|
||
|
|
- # some implied properties that might be used in the test suite.
|
||
|
|
- # Make sure we normalize to a specific product as well so that
|
||
|
|
- # when we load templated content it is correct.
|
||
|
|
- rule_path = get_rule_dir_yaml(dirpath)
|
||
|
|
- rule = RuleYAML.from_yaml(rule_path, product_yaml)
|
||
|
|
- rule.normalize(product)
|
||
|
|
-
|
||
|
|
- # Note that most places would check prodtype, but we don't care
|
||
|
|
- # about that here: if the rule is available to the product, we
|
||
|
|
- # load and parse it anyways as we have no knowledge of the
|
||
|
|
- # top-level profile or rule passed into the test suite.
|
||
|
|
- prodtypes = parse_prodtype(rule.prodtype)
|
||
|
|
-
|
||
|
|
- # Our local copy of env_yaml needs some properties from rule.yml
|
||
|
|
- # for completeness.
|
||
|
|
- local_env_yaml = dict()
|
||
|
|
- local_env_yaml.update(product_yaml)
|
||
|
|
- local_env_yaml['rule_id'] = rule.id_
|
||
|
|
- local_env_yaml['rule_title'] = rule.title
|
||
|
|
- local_env_yaml['products'] = prodtypes
|
||
|
|
+ # Load the rule and its environment
|
||
|
|
+ rule, local_env_yaml = load_rule_and_env(dirpath, product_yaml, product)
|
||
|
|
|
||
|
|
# Create the destination directory.
|
||
|
|
dest_path = os.path.join(tmpdir, rule.id_)
|
||
|
|
@@ -473,21 +498,66 @@ def iterate_over_rules(product=None):
|
||
|
|
id -- full rule id as it is present in datastream
|
||
|
|
short_id -- short rule ID, the same as basename of the directory
|
||
|
|
containing the test scenarios in Bash
|
||
|
|
- files -- list of executable .sh files in the "tests" directory
|
||
|
|
+ files -- list of executable .sh files in the uploaded tarball
|
||
|
|
"""
|
||
|
|
+
|
||
|
|
+ # Here we need to perform some magic to handle parsing the rule (from a
|
||
|
|
+ # product perspective) and loading any templated tests. In particular,
|
||
|
|
+ # identifying which tests to potentially run involves invoking the
|
||
|
|
+ # templating engine.
|
||
|
|
+ #
|
||
|
|
+ # Begin by loading context about our execution environment, if any.
|
||
|
|
+ product_yaml = get_product_context(product)
|
||
|
|
+
|
||
|
|
+ # Initialize a mock template_builder.
|
||
|
|
+ empty = "/ssgts/empty/placeholder"
|
||
|
|
+ template_builder = ssg.templates.Builder(product_yaml, empty,
|
||
|
|
+ _SHARED_TEMPLATES, empty, empty)
|
||
|
|
+
|
||
|
|
for dirpath, dirnames, filenames in walk_through_benchmark_dirs(product):
|
||
|
|
if "rule.yml" in filenames and "tests" in dirnames:
|
||
|
|
short_rule_id = os.path.basename(dirpath)
|
||
|
|
+
|
||
|
|
+ # Load the rule itself to check for a template.
|
||
|
|
+ rule, local_env_yaml = load_rule_and_env(dirpath, product_yaml, product)
|
||
|
|
+
|
||
|
|
+ # All tests is a mapping from path (in the tarball) to contents
|
||
|
|
+ # of the test case. This is necessary because later code (which
|
||
|
|
+ # attempts to parse headers from the test case) don't have easy
|
||
|
|
+ # access to templated content. By reading it and returning it
|
||
|
|
+ # here, we can save later code from having to understand the
|
||
|
|
+ # templating system.
|
||
|
|
+ all_tests = dict()
|
||
|
|
+
|
||
|
|
+ # Start
|
||
|
|
+ if rule.template:
|
||
|
|
+ templated_tests = template_builder.get_all_tests(
|
||
|
|
+ rule.id_, rule.template, local_env_yaml)
|
||
|
|
+ all_tests.update(templated_tests)
|
||
|
|
+
|
||
|
|
+ # Add additional tests from the local rule directory. Note that,
|
||
|
|
+ # like the behavior in template_tests, this will overwrite any
|
||
|
|
+ # templated tests with the same file name.
|
||
|
|
tests_dir = os.path.join(dirpath, "tests")
|
||
|
|
tests_dir_files = os.listdir(tests_dir)
|
||
|
|
+ for test_case in tests_dir_files:
|
||
|
|
+ test_path = os.path.join(tests_dir, test_case)
|
||
|
|
+ if os.path.isdir(test_path):
|
||
|
|
+ continue
|
||
|
|
+
|
||
|
|
+ with open(test_path) as fp:
|
||
|
|
+ all_tests[test_case] = fp.read()
|
||
|
|
+
|
||
|
|
# Filter out everything except the shell test scenarios.
|
||
|
|
# Other files in rule directories are editor swap files
|
||
|
|
# or other content than a test case.
|
||
|
|
- scripts = filter(lambda x: x.endswith(".sh"), tests_dir_files)
|
||
|
|
+ allowed_scripts = filter(lambda x: x.endswith(".sh"), all_tests)
|
||
|
|
+ content_mapping = {x: all_tests[x] for x in allowed_scripts}
|
||
|
|
+
|
||
|
|
full_rule_id = OSCAP_RULE + short_rule_id
|
||
|
|
result = Rule(
|
||
|
|
directory=tests_dir, id=full_rule_id, short_id=short_rule_id,
|
||
|
|
- files=scripts)
|
||
|
|
+ files=content_mapping)
|
||
|
|
yield result
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
From 84014297a6881610e682ec7d54eca658b6ff127a Mon Sep 17 00:00:00 2001
|
||
|
|
From: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
Date: Wed, 7 Jul 2021 07:05:13 -0400
|
||
|
|
Subject: [PATCH 07/19] Update rule execution to support templated tests
|
||
|
|
|
||
|
|
We needed to make a few small changes to rule-based test execution in
|
||
|
|
order for it to correctly understand templated tests:
|
||
|
|
|
||
|
|
1. Product selection (from the CLI) needs to be passed down into the
|
||
|
|
iterate_over_rules helper function.
|
||
|
|
2. Scenario loading needs to have knowledge of the returned test data.
|
||
|
|
3. Parsing of parameters needs to occur from the returned test data
|
||
|
|
rather than attempting to re-read it ourselves.
|
||
|
|
|
||
|
|
My assumption is that this will suffice for combined, since that method
|
||
|
|
of operation uses this method internally.
|
||
|
|
|
||
|
|
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
---
|
||
|
|
tests/ssg_test_suite/rule.py | 30 +++++++++++++++---------------
|
||
|
|
1 file changed, 15 insertions(+), 15 deletions(-)
|
||
|
|
|
||
|
|
diff --git a/tests/ssg_test_suite/rule.py b/tests/ssg_test_suite/rule.py
|
||
|
|
index 9f390749077..5ad7eb8ed27 100644
|
||
|
|
--- a/tests/ssg_test_suite/rule.py
|
||
|
|
+++ b/tests/ssg_test_suite/rule.py
|
||
|
|
@@ -24,7 +24,7 @@
|
||
|
|
|
||
|
|
|
||
|
|
Scenario = collections.namedtuple(
|
||
|
|
- "Scenario", ["script", "context", "script_params"])
|
||
|
|
+ "Scenario", ["script", "context", "script_params", "contents"])
|
||
|
|
|
||
|
|
|
||
|
|
def get_viable_profiles(selected_profiles, datastream, benchmark, script=None):
|
||
|
|
@@ -250,7 +250,7 @@ def _prepare_environment(self, scenarios_by_rule):
|
||
|
|
|
||
|
|
def _get_rules_to_test(self, target):
|
||
|
|
rules_to_test = []
|
||
|
|
- for rule in common.iterate_over_rules():
|
||
|
|
+ for rule in common.iterate_over_rules(self.test_env.product):
|
||
|
|
if not self._rule_should_be_tested(rule, target):
|
||
|
|
continue
|
||
|
|
if not xml_operations.find_rule_in_benchmark(
|
||
|
|
@@ -300,7 +300,7 @@ def _modify_parameters(self, script, params):
|
||
|
|
.format(OSCAP_PROFILE_ALL_ID, script))
|
||
|
|
return params
|
||
|
|
|
||
|
|
- def _parse_parameters(self, script):
|
||
|
|
+ def _parse_parameters(self, script_content):
|
||
|
|
"""Parse parameters from script header"""
|
||
|
|
params = {'profiles': [],
|
||
|
|
'templates': [],
|
||
|
|
@@ -309,16 +309,15 @@ def _parse_parameters(self, script):
|
||
|
|
'remediation': ['all'],
|
||
|
|
'variables': [],
|
||
|
|
}
|
||
|
|
- with open(script, 'r') as script_file:
|
||
|
|
- script_content = script_file.read()
|
||
|
|
- for parameter in params:
|
||
|
|
- found = re.search(r'^# {0} = (.*)$'.format(parameter),
|
||
|
|
- script_content,
|
||
|
|
- re.MULTILINE)
|
||
|
|
- if found is None:
|
||
|
|
- continue
|
||
|
|
- splitted = found.group(1).split(',')
|
||
|
|
- params[parameter] = [value.strip() for value in splitted]
|
||
|
|
+
|
||
|
|
+ for parameter in params:
|
||
|
|
+ found = re.search(r'^# {0} = (.*)$'.format(parameter),
|
||
|
|
+ script_content, re.MULTILINE)
|
||
|
|
+ if found is None:
|
||
|
|
+ continue
|
||
|
|
+ splitted = found.group(1).split(',')
|
||
|
|
+ params[parameter] = [value.strip() for value in splitted]
|
||
|
|
+
|
||
|
|
return params
|
||
|
|
|
||
|
|
def _get_scenarios(self, rule_dir, scripts, scenarios_regex, benchmark_cpes):
|
||
|
|
@@ -331,6 +330,7 @@ def _get_scenarios(self, rule_dir, scripts, scenarios_regex, benchmark_cpes):
|
||
|
|
|
||
|
|
scenarios = []
|
||
|
|
for script in scripts:
|
||
|
|
+ script_contents = scripts[script]
|
||
|
|
if scenarios_regex is not None:
|
||
|
|
if scenarios_pattern.match(script) is None:
|
||
|
|
logging.debug("Skipping script %s - it did not match "
|
||
|
|
@@ -338,10 +338,10 @@ def _get_scenarios(self, rule_dir, scripts, scenarios_regex, benchmark_cpes):
|
||
|
|
continue
|
||
|
|
script_context = _get_script_context(script)
|
||
|
|
if script_context is not None:
|
||
|
|
- script_params = self._parse_parameters(os.path.join(rule_dir, script))
|
||
|
|
+ script_params = self._parse_parameters(script_contents)
|
||
|
|
script_params = self._modify_parameters(script, script_params)
|
||
|
|
if common.matches_platform(script_params["platform"], benchmark_cpes):
|
||
|
|
- scenarios += [Scenario(script, script_context, script_params)]
|
||
|
|
+ scenarios += [Scenario(script, script_context, script_params, script_contents)]
|
||
|
|
else:
|
||
|
|
logging.warning("Script %s is not applicable on given platform" % script)
|
||
|
|
|
||
|
|
|
||
|
|
From c763af671037cb9bfd5bfd5bd24f5efe9e4fc4c1 Mon Sep 17 00:00:00 2001
|
||
|
|
From: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
Date: Wed, 7 Jul 2021 07:22:47 -0400
|
||
|
|
Subject: [PATCH 08/19] Make sure we read all test contents with Jinja
|
||
|
|
|
||
|
|
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
---
|
||
|
|
tests/ssg_test_suite/common.py | 3 +--
|
||
|
|
1 file changed, 1 insertion(+), 2 deletions(-)
|
||
|
|
|
||
|
|
diff --git a/tests/ssg_test_suite/common.py b/tests/ssg_test_suite/common.py
|
||
|
|
index b357f816027..865c8a8b988 100644
|
||
|
|
--- a/tests/ssg_test_suite/common.py
|
||
|
|
+++ b/tests/ssg_test_suite/common.py
|
||
|
|
@@ -545,8 +545,7 @@ def iterate_over_rules(product=None):
|
||
|
|
if os.path.isdir(test_path):
|
||
|
|
continue
|
||
|
|
|
||
|
|
- with open(test_path) as fp:
|
||
|
|
- all_tests[test_case] = fp.read()
|
||
|
|
+ all_tests[test_case] = process_file(test_path, local_env_yaml)
|
||
|
|
|
||
|
|
# Filter out everything except the shell test scenarios.
|
||
|
|
# Other files in rule directories are editor swap files
|
||
|
|
|
||
|
|
From 1234addaa4f776e3c814192c0aaf8f2137a74481 Mon Sep 17 00:00:00 2001
|
||
|
|
From: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
Date: Thu, 8 Jul 2021 07:44:16 -0400
|
||
|
|
Subject: [PATCH 09/19] Fix pep8 issues
|
||
|
|
|
||
|
|
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
---
|
||
|
|
ssg/templates.py | 5 +++--
|
||
|
|
tests/ssg_test_suite/common.py | 5 +++--
|
||
|
|
2 files changed, 6 insertions(+), 4 deletions(-)
|
||
|
|
|
||
|
|
diff --git a/ssg/templates.py b/ssg/templates.py
|
||
|
|
index 0a39c3ba094..818b1f79f68 100644
|
||
|
|
--- a/ssg/templates.py
|
||
|
|
+++ b/ssg/templates.py
|
||
|
|
@@ -201,7 +201,8 @@ def build_lang(
|
||
|
|
return
|
||
|
|
|
||
|
|
filled_template = self.build_lang_file(rule_id, template_name,
|
||
|
|
- template_vars, lang, local_env_yaml)
|
||
|
|
+ template_vars, lang,
|
||
|
|
+ local_env_yaml)
|
||
|
|
|
||
|
|
ext = lang_to_ext_map[lang]
|
||
|
|
output_file_name = rule_id + ext
|
||
|
|
@@ -331,7 +332,7 @@ def get_lang_for_rule(self, rule_id, rule_title, template, language):
|
||
|
|
local_env_yaml["products"] = self.env_yaml["product"]
|
||
|
|
|
||
|
|
return self.build_lang_file(rule_id, template_name, template_vars,
|
||
|
|
- language, local_env_yaml)
|
||
|
|
+ language, local_env_yaml)
|
||
|
|
|
||
|
|
def build_extra_ovals(self):
|
||
|
|
declaration_path = os.path.join(self.templates_dir, "extra_ovals.yml")
|
||
|
|
diff --git a/tests/ssg_test_suite/common.py b/tests/ssg_test_suite/common.py
|
||
|
|
index 865c8a8b988..977e9f52c24 100644
|
||
|
|
--- a/tests/ssg_test_suite/common.py
|
||
|
|
+++ b/tests/ssg_test_suite/common.py
|
||
|
|
@@ -338,7 +338,8 @@ def template_tests(product=None):
|
||
|
|
# Initialize a mock template_builder.
|
||
|
|
empty = "/ssgts/empty/placeholder"
|
||
|
|
template_builder = ssg.templates.Builder(product_yaml, empty,
|
||
|
|
- _SHARED_TEMPLATES, empty, empty)
|
||
|
|
+ _SHARED_TEMPLATES, empty,
|
||
|
|
+ empty)
|
||
|
|
|
||
|
|
# Note that we're not exactly copying 1-for-1 the contents of the
|
||
|
|
# directory structure into the temporary one. Instead we want a
|
||
|
|
@@ -512,7 +513,7 @@ def iterate_over_rules(product=None):
|
||
|
|
# Initialize a mock template_builder.
|
||
|
|
empty = "/ssgts/empty/placeholder"
|
||
|
|
template_builder = ssg.templates.Builder(product_yaml, empty,
|
||
|
|
- _SHARED_TEMPLATES, empty, empty)
|
||
|
|
+ _SHARED_TEMPLATES, empty, empty)
|
||
|
|
|
||
|
|
for dirpath, dirnames, filenames in walk_through_benchmark_dirs(product):
|
||
|
|
if "rule.yml" in filenames and "tests" in dirnames:
|
||
|
|
|
||
|
|
From 81fcdf3e18f83da44b697ef7f47759a8eadd9854 Mon Sep 17 00:00:00 2001
|
||
|
|
From: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
Date: Wed, 14 Jul 2021 08:06:57 -0400
|
||
|
|
Subject: [PATCH 10/19] Split template_tests into template_rule_tests
|
||
|
|
|
||
|
|
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
---
|
||
|
|
tests/ssg_test_suite/common.py | 146 +++++++++++++++++----------------
|
||
|
|
1 file changed, 77 insertions(+), 69 deletions(-)
|
||
|
|
|
||
|
|
diff --git a/tests/ssg_test_suite/common.py b/tests/ssg_test_suite/common.py
|
||
|
|
index 977e9f52c24..4e77bd888fe 100644
|
||
|
|
--- a/tests/ssg_test_suite/common.py
|
||
|
|
+++ b/tests/ssg_test_suite/common.py
|
||
|
|
@@ -320,6 +320,82 @@ def load_rule_and_env(rule_dir_path, env_yaml, product=None):
|
||
|
|
return rule, local_env_yaml
|
||
|
|
|
||
|
|
|
||
|
|
+def template_rule_tests(product, product_yaml, template_builder, tmpdir, dirpath):
|
||
|
|
+ """
|
||
|
|
+ For a given rule directory, templates all contained tests into the output
|
||
|
|
+ (tmpdir) directory.
|
||
|
|
+ """
|
||
|
|
+
|
||
|
|
+ # Load the rule and its environment
|
||
|
|
+ rule, local_env_yaml = load_rule_and_env(dirpath, product_yaml, product)
|
||
|
|
+
|
||
|
|
+ # Create the destination directory.
|
||
|
|
+ dest_path = os.path.join(tmpdir, rule.id_)
|
||
|
|
+ os.mkdir(dest_path)
|
||
|
|
+
|
||
|
|
+ # The priority order is rule-specific tests over templated tests.
|
||
|
|
+ # That is, for any test under rule_id/tests with a name matching a
|
||
|
|
+ # test under shared/templates/<template_name>/tests/, the former
|
||
|
|
+ # will preferred. This means we need to process templates first,
|
||
|
|
+ # so they'll be overwritten later if necessary.
|
||
|
|
+ if rule.template:
|
||
|
|
+ templated_tests = template_builder.get_all_tests(rule.id_, rule.template,
|
||
|
|
+ local_env_yaml)
|
||
|
|
+
|
||
|
|
+ for relative_path in templated_tests:
|
||
|
|
+ output_path = os.path.join(dest_path, relative_path)
|
||
|
|
+
|
||
|
|
+ # If there's a separator in the file name, it means we
|
||
|
|
+ # have nested directories to deal with.
|
||
|
|
+ if os.path.sep in relative_path:
|
||
|
|
+ parts = os.path.split(relative_path)[:-1]
|
||
|
|
+ for subdir_index in range(len(parts)):
|
||
|
|
+ # We need to expand all directories in correct
|
||
|
|
+ # order, preserving any previous directories (as
|
||
|
|
+ # they're nested). Use the star operator to splat
|
||
|
|
+ # array parts into arguments to os.path.join(...).
|
||
|
|
+ new_directory = os.path.join(dest_path, *parts[:subdir_index])
|
||
|
|
+ os.mkdir(new_directory)
|
||
|
|
+
|
||
|
|
+ # Write out the test content to the desired location on
|
||
|
|
+ # disk.
|
||
|
|
+ with open(output_path, 'w') as output_fp:
|
||
|
|
+ test_content = templated_tests[relative_path]
|
||
|
|
+ print(test_content, file=output_fp)
|
||
|
|
+
|
||
|
|
+ # Walk the test directory, writing all tests into the output
|
||
|
|
+ # directory, recursively.
|
||
|
|
+ tests_dir_path = os.path.join(dirpath, "tests")
|
||
|
|
+ tests_dir_path = os.path.abspath(tests_dir_path)
|
||
|
|
+ for dirpath, dirnames, filenames in os.walk(tests_dir_path):
|
||
|
|
+ for dirname in dirnames:
|
||
|
|
+ # We want to recreate the correct path under the temporary
|
||
|
|
+ # directory. Resolve it to a relative path from the tests/
|
||
|
|
+ # directory.
|
||
|
|
+ dir_path = _rel_abs_path(os.path.join(dirpath, dirname), tests_dir_path)
|
||
|
|
+ assert '../' not in dir_path
|
||
|
|
+ tmp_dir_path = os.path.join(dest_path, dir_path)
|
||
|
|
+ os.mkdir(tmp_dir_path)
|
||
|
|
+
|
||
|
|
+ for filename in filenames:
|
||
|
|
+ # We want to recreate the correct path under the temporary
|
||
|
|
+ # directory. Resolve it to a relative path from the tests/
|
||
|
|
+ # directory. Assumption: directories should be created
|
||
|
|
+ # prior to recursing into them, so we don't need to handle
|
||
|
|
+ # if a file's parent directory doesn't yet exist under the
|
||
|
|
+ # destination.
|
||
|
|
+ src_test_path = os.path.join(dirpath, filename)
|
||
|
|
+ rel_test_path = _rel_abs_path(src_test_path, tests_dir_path)
|
||
|
|
+ dest_test_path = os.path.join(dest_path, rel_test_path)
|
||
|
|
+
|
||
|
|
+ # Rather than performing an OS-level copy, we need to
|
||
|
|
+ # first parse the test with jinja and then write it back
|
||
|
|
+ # out to the destination.
|
||
|
|
+ parsed_test = process_file(src_test_path, local_env_yaml)
|
||
|
|
+ with open(dest_test_path, 'w') as output_fp:
|
||
|
|
+ print(parsed_test, file=output_fp)
|
||
|
|
+
|
||
|
|
+
|
||
|
|
def template_tests(product=None):
|
||
|
|
"""
|
||
|
|
Create a temporary directory with test cases parsed via jinja using
|
||
|
|
@@ -352,75 +428,7 @@ def template_tests(product=None):
|
||
|
|
if "tests" not in dirnames or not is_rule_dir(dirpath):
|
||
|
|
continue
|
||
|
|
|
||
|
|
- # Load the rule and its environment
|
||
|
|
- rule, local_env_yaml = load_rule_and_env(dirpath, product_yaml, product)
|
||
|
|
-
|
||
|
|
- # Create the destination directory.
|
||
|
|
- dest_path = os.path.join(tmpdir, rule.id_)
|
||
|
|
- os.mkdir(dest_path)
|
||
|
|
-
|
||
|
|
- # The priority order is rule-specific tests over templated tests.
|
||
|
|
- # That is, for any test under rule_id/tests with a name matching a
|
||
|
|
- # test under shared/templates/<template_name>/tests/, the former
|
||
|
|
- # will preferred. This means we need to process templates first,
|
||
|
|
- # so they'll be overwritten later if necessary.
|
||
|
|
- if rule.template:
|
||
|
|
- templated_tests = template_builder.get_all_tests(
|
||
|
|
- rule.id_, rule.template, local_env_yaml)
|
||
|
|
-
|
||
|
|
- for relative_path in templated_tests:
|
||
|
|
- output_path = os.path.join(dest_path, relative_path)
|
||
|
|
-
|
||
|
|
- # If there's a separator in the file name, it means we
|
||
|
|
- # have nested directories to deal with.
|
||
|
|
- if os.path.sep in relative_path:
|
||
|
|
- parts = os.path.split(relative_path)[:-1]
|
||
|
|
- for subdir_index in range(len(parts)):
|
||
|
|
- # We need to expand all directories in correct
|
||
|
|
- # order, preserving any previous directories (as
|
||
|
|
- # they're nested). Use the star operator to splat
|
||
|
|
- # array parts into arguments to os.path.join(...).
|
||
|
|
- new_directory = os.path.join(dest_path, *parts[:subdir_index])
|
||
|
|
- os.mkdir(new_directory)
|
||
|
|
-
|
||
|
|
- # Write out the test content to the desired location on
|
||
|
|
- # disk.
|
||
|
|
- with open(output_path, 'w') as output_fp:
|
||
|
|
- test_content = templated_tests[relative_path]
|
||
|
|
- print(test_content, file=output_fp)
|
||
|
|
-
|
||
|
|
- # Walk the test directory, writing all tests into the output
|
||
|
|
- # directory, recursively.
|
||
|
|
- tests_dir_path = os.path.join(dirpath, "tests")
|
||
|
|
- tests_dir_path = os.path.abspath(tests_dir_path)
|
||
|
|
- for dirpath, dirnames, filenames in os.walk(tests_dir_path):
|
||
|
|
- for dirname in dirnames:
|
||
|
|
- # We want to recreate the correct path under the temporary
|
||
|
|
- # directory. Resolve it to a relative path from the tests/
|
||
|
|
- # directory.
|
||
|
|
- dir_path = _rel_abs_path(os.path.join(dirpath, dirname), tests_dir_path)
|
||
|
|
- assert '../' not in dir_path
|
||
|
|
- tmp_dir_path = os.path.join(dest_path, dir_path)
|
||
|
|
- os.mkdir(tmp_dir_path)
|
||
|
|
-
|
||
|
|
- for filename in filenames:
|
||
|
|
- # We want to recreate the correct path under the temporary
|
||
|
|
- # directory. Resolve it to a relative path from the tests/
|
||
|
|
- # directory. Assumption: directories should be created
|
||
|
|
- # prior to recursing into them, so we don't need to handle
|
||
|
|
- # if a file's parent directory doesn't yet exist under the
|
||
|
|
- # destination.
|
||
|
|
- src_test_path = os.path.join(dirpath, filename)
|
||
|
|
- rel_test_path = _rel_abs_path(src_test_path, tests_dir_path)
|
||
|
|
- dest_test_path = os.path.join(dest_path, rel_test_path)
|
||
|
|
-
|
||
|
|
- # Rather than performing an OS-level copy, we need to
|
||
|
|
- # first parse the test with jinja and then write it back
|
||
|
|
- # out to the destination.
|
||
|
|
- parsed_test = process_file(src_test_path, local_env_yaml)
|
||
|
|
- with open(dest_test_path, 'w') as output_fp:
|
||
|
|
- print(parsed_test, file=output_fp)
|
||
|
|
-
|
||
|
|
+ template_rule_tests(product, product_yaml, template_builder, tmpdir, dirpath)
|
||
|
|
except Exception as exp:
|
||
|
|
shutil.rmtree(tmpdir, ignore_errors=True)
|
||
|
|
raise exp
|
||
|
|
|
||
|
|
From 9bd6bf1bb5a51a69dc30d4d6eaa2e159c2ac0446 Mon Sep 17 00:00:00 2001
|
||
|
|
From: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
Date: Wed, 14 Jul 2021 08:23:27 -0400
|
||
|
|
Subject: [PATCH 11/19] Refactor template_rule_tests into write_* helpers
|
||
|
|
|
||
|
|
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
---
|
||
|
|
tests/ssg_test_suite/common.py | 93 ++++++++++++++++++----------------
|
||
|
|
1 file changed, 50 insertions(+), 43 deletions(-)
|
||
|
|
|
||
|
|
diff --git a/tests/ssg_test_suite/common.py b/tests/ssg_test_suite/common.py
|
||
|
|
index 4e77bd888fe..1f3cad807e6 100644
|
||
|
|
--- a/tests/ssg_test_suite/common.py
|
||
|
|
+++ b/tests/ssg_test_suite/common.py
|
||
|
|
@@ -320,49 +320,27 @@ def load_rule_and_env(rule_dir_path, env_yaml, product=None):
|
||
|
|
return rule, local_env_yaml
|
||
|
|
|
||
|
|
|
||
|
|
-def template_rule_tests(product, product_yaml, template_builder, tmpdir, dirpath):
|
||
|
|
- """
|
||
|
|
- For a given rule directory, templates all contained tests into the output
|
||
|
|
- (tmpdir) directory.
|
||
|
|
- """
|
||
|
|
-
|
||
|
|
- # Load the rule and its environment
|
||
|
|
- rule, local_env_yaml = load_rule_and_env(dirpath, product_yaml, product)
|
||
|
|
-
|
||
|
|
- # Create the destination directory.
|
||
|
|
- dest_path = os.path.join(tmpdir, rule.id_)
|
||
|
|
- os.mkdir(dest_path)
|
||
|
|
-
|
||
|
|
- # The priority order is rule-specific tests over templated tests.
|
||
|
|
- # That is, for any test under rule_id/tests with a name matching a
|
||
|
|
- # test under shared/templates/<template_name>/tests/, the former
|
||
|
|
- # will preferred. This means we need to process templates first,
|
||
|
|
- # so they'll be overwritten later if necessary.
|
||
|
|
- if rule.template:
|
||
|
|
- templated_tests = template_builder.get_all_tests(rule.id_, rule.template,
|
||
|
|
- local_env_yaml)
|
||
|
|
-
|
||
|
|
- for relative_path in templated_tests:
|
||
|
|
- output_path = os.path.join(dest_path, relative_path)
|
||
|
|
-
|
||
|
|
- # If there's a separator in the file name, it means we
|
||
|
|
- # have nested directories to deal with.
|
||
|
|
- if os.path.sep in relative_path:
|
||
|
|
- parts = os.path.split(relative_path)[:-1]
|
||
|
|
- for subdir_index in range(len(parts)):
|
||
|
|
- # We need to expand all directories in correct
|
||
|
|
- # order, preserving any previous directories (as
|
||
|
|
- # they're nested). Use the star operator to splat
|
||
|
|
- # array parts into arguments to os.path.join(...).
|
||
|
|
- new_directory = os.path.join(dest_path, *parts[:subdir_index])
|
||
|
|
- os.mkdir(new_directory)
|
||
|
|
-
|
||
|
|
- # Write out the test content to the desired location on
|
||
|
|
- # disk.
|
||
|
|
- with open(output_path, 'w') as output_fp:
|
||
|
|
- test_content = templated_tests[relative_path]
|
||
|
|
- print(test_content, file=output_fp)
|
||
|
|
-
|
||
|
|
+def write_rule_templated_tests(dest_path, relative_path, test_content):
|
||
|
|
+ output_path = os.path.join(dest_path, relative_path)
|
||
|
|
+
|
||
|
|
+ # If there's a separator in the file name, it means we have nested
|
||
|
|
+ # directories to deal with.
|
||
|
|
+ if os.path.sep in relative_path:
|
||
|
|
+ parts = os.path.split(relative_path)[:-1]
|
||
|
|
+ for subdir_index in range(len(parts)):
|
||
|
|
+ # We need to expand all directories in the correct order,
|
||
|
|
+ # preserving any previous directories (as they're nested).
|
||
|
|
+ # Use the star operator to splat array parts into arguments
|
||
|
|
+ # to os.path.join(...).
|
||
|
|
+ new_directory = os.path.join(dest_path, *parts[:subdir_index])
|
||
|
|
+ os.mkdir(new_directory)
|
||
|
|
+
|
||
|
|
+ # Write out the test content to the desired location on disk.
|
||
|
|
+ with open(output_path, 'w') as output_fp:
|
||
|
|
+ print(test_content, file=output_fp)
|
||
|
|
+
|
||
|
|
+
|
||
|
|
+def write_rule_dir_tests(local_env_yaml, dest_path, dirpath):
|
||
|
|
# Walk the test directory, writing all tests into the output
|
||
|
|
# directory, recursively.
|
||
|
|
tests_dir_path = os.path.join(dirpath, "tests")
|
||
|
|
@@ -396,6 +374,35 @@ def template_rule_tests(product, product_yaml, template_builder, tmpdir, dirpath
|
||
|
|
print(parsed_test, file=output_fp)
|
||
|
|
|
||
|
|
|
||
|
|
+def template_rule_tests(product, product_yaml, template_builder, tmpdir, dirpath):
|
||
|
|
+ """
|
||
|
|
+ For a given rule directory, templates all contained tests into the output
|
||
|
|
+ (tmpdir) directory.
|
||
|
|
+ """
|
||
|
|
+
|
||
|
|
+ # Load the rule and its environment
|
||
|
|
+ rule, local_env_yaml = load_rule_and_env(dirpath, product_yaml, product)
|
||
|
|
+
|
||
|
|
+ # Create the destination directory.
|
||
|
|
+ dest_path = os.path.join(tmpdir, rule.id_)
|
||
|
|
+ os.mkdir(dest_path)
|
||
|
|
+
|
||
|
|
+ # The priority order is rule-specific tests over templated tests.
|
||
|
|
+ # That is, for any test under rule_id/tests with a name matching a
|
||
|
|
+ # test under shared/templates/<template_name>/tests/, the former
|
||
|
|
+ # will preferred. This means we need to process templates first,
|
||
|
|
+ # so they'll be overwritten later if necessary.
|
||
|
|
+ if rule.template:
|
||
|
|
+ templated_tests = template_builder.get_all_tests(rule.id_, rule.template,
|
||
|
|
+ local_env_yaml)
|
||
|
|
+
|
||
|
|
+ for relative_path in templated_tests:
|
||
|
|
+ test_content = templated_tests[relative_path]
|
||
|
|
+ write_rule_templated_tests(dest_path, relative_path, test_content)
|
||
|
|
+
|
||
|
|
+ write_rule_dir_tests(local_env_yaml, dest_path, dirpath)
|
||
|
|
+
|
||
|
|
+
|
||
|
|
def template_tests(product=None):
|
||
|
|
"""
|
||
|
|
Create a temporary directory with test cases parsed via jinja using
|
||
|
|
|
||
|
|
From 5b74ff0aa8ef1b085ef9d6c0148283835a2917f8 Mon Sep 17 00:00:00 2001
|
||
|
|
From: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
Date: Wed, 14 Jul 2021 08:24:10 -0400
|
||
|
|
Subject: [PATCH 12/19] Remove unnecessary assertion
|
||
|
|
|
||
|
|
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
---
|
||
|
|
tests/ssg_test_suite/common.py | 1 -
|
||
|
|
1 file changed, 1 deletion(-)
|
||
|
|
|
||
|
|
diff --git a/tests/ssg_test_suite/common.py b/tests/ssg_test_suite/common.py
|
||
|
|
index 1f3cad807e6..9c232dcad02 100644
|
||
|
|
--- a/tests/ssg_test_suite/common.py
|
||
|
|
+++ b/tests/ssg_test_suite/common.py
|
||
|
|
@@ -351,7 +351,6 @@ def write_rule_dir_tests(local_env_yaml, dest_path, dirpath):
|
||
|
|
# directory. Resolve it to a relative path from the tests/
|
||
|
|
# directory.
|
||
|
|
dir_path = _rel_abs_path(os.path.join(dirpath, dirname), tests_dir_path)
|
||
|
|
- assert '../' not in dir_path
|
||
|
|
tmp_dir_path = os.path.join(dest_path, dir_path)
|
||
|
|
os.mkdir(tmp_dir_path)
|
||
|
|
|
||
|
|
|
||
|
|
From 0ab1b95de66d60cf135c985697d6269464777de9 Mon Sep 17 00:00:00 2001
|
||
|
|
From: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
Date: Wed, 14 Jul 2021 08:25:26 -0400
|
||
|
|
Subject: [PATCH 13/19] Remove unnecessary _rel_abs_path
|
||
|
|
|
||
|
|
As pointed out by Matej, the extra abspath is unnecessary prior to
|
||
|
|
calling relpath. Remove our helper and switch to calling relpath
|
||
|
|
directly.
|
||
|
|
|
||
|
|
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
---
|
||
|
|
tests/ssg_test_suite/common.py | 15 ++-------------
|
||
|
|
1 file changed, 2 insertions(+), 13 deletions(-)
|
||
|
|
|
||
|
|
diff --git a/tests/ssg_test_suite/common.py b/tests/ssg_test_suite/common.py
|
||
|
|
index 9c232dcad02..04117359203 100644
|
||
|
|
--- a/tests/ssg_test_suite/common.py
|
||
|
|
+++ b/tests/ssg_test_suite/common.py
|
||
|
|
@@ -253,17 +253,6 @@ def _make_file_root_owned(tarinfo):
|
||
|
|
return tarinfo
|
||
|
|
|
||
|
|
|
||
|
|
-def _rel_abs_path(current_path, base_path):
|
||
|
|
- """
|
||
|
|
- Return the value of the current path, relative to the base path, but
|
||
|
|
- resolving paths absolutely first. This helps when walking a nested
|
||
|
|
- directory structure and want to get the subtree relative to the original
|
||
|
|
- path
|
||
|
|
- """
|
||
|
|
- tmp_path = os.path.abspath(current_path)
|
||
|
|
- return os.path.relpath(current_path, base_path)
|
||
|
|
-
|
||
|
|
-
|
||
|
|
def get_product_context(product=None):
|
||
|
|
"""
|
||
|
|
Returns a product YAML context if any product is specified. Hard-coded to
|
||
|
|
@@ -350,7 +339,7 @@ def write_rule_dir_tests(local_env_yaml, dest_path, dirpath):
|
||
|
|
# We want to recreate the correct path under the temporary
|
||
|
|
# directory. Resolve it to a relative path from the tests/
|
||
|
|
# directory.
|
||
|
|
- dir_path = _rel_abs_path(os.path.join(dirpath, dirname), tests_dir_path)
|
||
|
|
+ dir_path = os.path.relpath(os.path.join(dirpath, dirname), tests_dir_path)
|
||
|
|
tmp_dir_path = os.path.join(dest_path, dir_path)
|
||
|
|
os.mkdir(tmp_dir_path)
|
||
|
|
|
||
|
|
@@ -362,7 +351,7 @@ def write_rule_dir_tests(local_env_yaml, dest_path, dirpath):
|
||
|
|
# if a file's parent directory doesn't yet exist under the
|
||
|
|
# destination.
|
||
|
|
src_test_path = os.path.join(dirpath, filename)
|
||
|
|
- rel_test_path = _rel_abs_path(src_test_path, tests_dir_path)
|
||
|
|
+ rel_test_path = os.path.relpath(src_test_path, tests_dir_path)
|
||
|
|
dest_test_path = os.path.join(dest_path, rel_test_path)
|
||
|
|
|
||
|
|
# Rather than performing an OS-level copy, we need to
|
||
|
|
|
||
|
|
From 0d2dadf00682efba465b5378803a68893dc62038 Mon Sep 17 00:00:00 2001
|
||
|
|
From: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
Date: Wed, 14 Jul 2021 14:23:59 -0400
|
||
|
|
Subject: [PATCH 14/19] Only template rules with variables
|
||
|
|
|
||
|
|
The audit_rules_privileged_commands_unix2_chkpwd rule lacks variables on
|
||
|
|
most products, in addition to its limited prodtype. Because the existing
|
||
|
|
test harness lacks understanding of prodtype (for including rules in the
|
||
|
|
tarball), we don't check it yet either. This causes issues when the
|
||
|
|
template is present but has empty variables on the particular product
|
||
|
|
(such as when this rule is executed under Ubuntu)
|
||
|
|
|
||
|
|
Ultimately we should probably check prodtype in the future, but it
|
||
|
|
outside the scope of this PR.
|
||
|
|
|
||
|
|
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
---
|
||
|
|
tests/ssg_test_suite/common.py | 4 ++--
|
||
|
|
1 file changed, 2 insertions(+), 2 deletions(-)
|
||
|
|
|
||
|
|
diff --git a/tests/ssg_test_suite/common.py b/tests/ssg_test_suite/common.py
|
||
|
|
index 04117359203..44105e3b7ae 100644
|
||
|
|
--- a/tests/ssg_test_suite/common.py
|
||
|
|
+++ b/tests/ssg_test_suite/common.py
|
||
|
|
@@ -380,7 +380,7 @@ def template_rule_tests(product, product_yaml, template_builder, tmpdir, dirpath
|
||
|
|
# test under shared/templates/<template_name>/tests/, the former
|
||
|
|
# will preferred. This means we need to process templates first,
|
||
|
|
# so they'll be overwritten later if necessary.
|
||
|
|
- if rule.template:
|
||
|
|
+ if rule.template and rule.template['vars']:
|
||
|
|
templated_tests = template_builder.get_all_tests(rule.id_, rule.template,
|
||
|
|
local_env_yaml)
|
||
|
|
|
||
|
|
@@ -534,7 +534,7 @@ def iterate_over_rules(product=None):
|
||
|
|
all_tests = dict()
|
||
|
|
|
||
|
|
# Start
|
||
|
|
- if rule.template:
|
||
|
|
+ if rule.template and rule.template['vars']:
|
||
|
|
templated_tests = template_builder.get_all_tests(
|
||
|
|
rule.id_, rule.template, local_env_yaml)
|
||
|
|
all_tests.update(templated_tests)
|
||
|
|
|
||
|
|
From e00262230d86b39d77dc1d75ffce5c048d5a4652 Mon Sep 17 00:00:00 2001
|
||
|
|
From: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
Date: Mon, 19 Jul 2021 09:06:02 -0400
|
||
|
|
Subject: [PATCH 15/19] Document new JINJA/Template testing scenarios
|
||
|
|
|
||
|
|
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
---
|
||
|
|
tests/README.md | 32 ++++++++++++++++++++++++++++++++
|
||
|
|
1 file changed, 32 insertions(+)
|
||
|
|
|
||
|
|
diff --git a/tests/README.md b/tests/README.md
|
||
|
|
index 6b5b4497baf..0b3dfabc6f5 100644
|
||
|
|
--- a/tests/README.md
|
||
|
|
+++ b/tests/README.md
|
||
|
|
@@ -147,6 +147,7 @@ the rule that `oscap` should return when the rule is evaluated.
|
||
|
|
very important to keep this naming form.
|
||
|
|
|
||
|
|
For example:
|
||
|
|
+
|
||
|
|
* `something.pass.sh`: Success scenario - script is expected to prepare machine
|
||
|
|
in such way that the rule is expected to pass.
|
||
|
|
* `something.fail.sh`: Fail scenario - script is expected to break machine so
|
||
|
|
@@ -200,6 +201,37 @@ Using `platform` and `variables` metadata:
|
||
|
|
echo "KerberosAuthentication $auth_enabled" >> /etc/ssh/sshd_config
|
||
|
|
```
|
||
|
|
|
||
|
|
+### Augmenting using Jinja macros
|
||
|
|
+
|
||
|
|
+Each scenario script is processed under the same jinja context as the
|
||
|
|
+corresponding OVAL and remediation content. This means that product-specific
|
||
|
|
+information is known to the scenario scripts at upload time (for example,
|
||
|
|
+`{{{ grub2_boot_path }}}`), allowing them to work across products. This
|
||
|
|
+also means Jinja macros such as `{{{ bash_package_install(...) }}}` work to
|
||
|
|
+install/remove specific packages during the course of testing (such as, if
|
||
|
|
+it is desired to both install and remove a package in the same scenario for
|
||
|
|
+the `package_installed` rules).
|
||
|
|
+
|
||
|
|
+Note that this does have some limitations: knowledge of the profile (and the
|
||
|
|
+variables it has and the values they take) is still not provided to the test
|
||
|
|
+scenario. The above `# profiles` or `# variables` directives will still have
|
||
|
|
+to be used to add any profile-specific information.
|
||
|
|
+
|
||
|
|
+### Augmenting using `shared/templates`
|
||
|
|
+
|
||
|
|
+Additionally, we have enabled test scenarios located under the templated
|
||
|
|
+directory, `shared/templates/.../tests`. Unlike with build-time content,
|
||
|
|
+`tests` does not need to be located in the template's manifest (at
|
||
|
|
+`template.yml`). Instead, SSGTS will automatically parse each rule and
|
||
|
|
+prefer rule-directory-specific test scenarios over any templated scenarios
|
||
|
|
+that the rule uses. (E.g., if `installed.pass.sh` is present in the
|
||
|
|
+template `package_installed` and in the `tests/` subdirectory of the rule
|
||
|
|
+directory, the latter takes precedence over the former).
|
||
|
|
+
|
||
|
|
+In addition to the Jinja context described above, the contents of the template
|
||
|
|
+variables (after processing in `template.py`) are also available to the
|
||
|
|
+test scenario. This enables template-specific checking.
|
||
|
|
+
|
||
|
|
## Example of adding new test scenarios
|
||
|
|
|
||
|
|
Let's add test scenarios for rule `accounts_password_minlen_login_defs`.
|
||
|
|
|
||
|
|
From c696ebf0001bd6bcced40aafea58cfbc6b870cc1 Mon Sep 17 00:00:00 2001
|
||
|
|
From: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
Date: Tue, 27 Jul 2021 08:17:47 -0400
|
||
|
|
Subject: [PATCH 16/19] Correctly handle generation of templated tests
|
||
|
|
|
||
|
|
When rules lacked tests inside the rule directory but had them via a
|
||
|
|
template, SSGTS would ignore them and claim the rule wasn't found. Fix
|
||
|
|
this bug to allow template-only tests.
|
||
|
|
|
||
|
|
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
---
|
||
|
|
tests/ssg_test_suite/common.py | 30 ++++++++++++++++++++++--------
|
||
|
|
1 file changed, 22 insertions(+), 8 deletions(-)
|
||
|
|
|
||
|
|
diff --git a/tests/ssg_test_suite/common.py b/tests/ssg_test_suite/common.py
|
||
|
|
index 44105e3b7ae..0cb5451e31b 100644
|
||
|
|
--- a/tests/ssg_test_suite/common.py
|
||
|
|
+++ b/tests/ssg_test_suite/common.py
|
||
|
|
@@ -334,6 +334,13 @@ def write_rule_dir_tests(local_env_yaml, dest_path, dirpath):
|
||
|
|
# directory, recursively.
|
||
|
|
tests_dir_path = os.path.join(dirpath, "tests")
|
||
|
|
tests_dir_path = os.path.abspath(tests_dir_path)
|
||
|
|
+
|
||
|
|
+ # Note that the tests/ directory may not always exist any more. In
|
||
|
|
+ # particular, when a rule uses a template, tests may be present there
|
||
|
|
+ # but not present in the actual rule directory.
|
||
|
|
+ if not os.path.exists(tests_dir_path):
|
||
|
|
+ return
|
||
|
|
+
|
||
|
|
for dirpath, dirnames, filenames in os.walk(tests_dir_path):
|
||
|
|
for dirname in dirnames:
|
||
|
|
# We want to recreate the correct path under the temporary
|
||
|
|
@@ -420,7 +427,7 @@ def template_tests(product=None):
|
||
|
|
# /group_a/rule_a/tests/something.pass.sh -> /rule_a/something.pass.sh
|
||
|
|
for dirpath, dirnames, _ in walk_through_benchmark_dirs(product):
|
||
|
|
# Skip anything that isn't obviously a rule.
|
||
|
|
- if "tests" not in dirnames or not is_rule_dir(dirpath):
|
||
|
|
+ if not is_rule_dir(dirpath):
|
||
|
|
continue
|
||
|
|
|
||
|
|
template_rule_tests(product, product_yaml, template_builder, tmpdir, dirpath)
|
||
|
|
@@ -519,7 +526,7 @@ def iterate_over_rules(product=None):
|
||
|
|
_SHARED_TEMPLATES, empty, empty)
|
||
|
|
|
||
|
|
for dirpath, dirnames, filenames in walk_through_benchmark_dirs(product):
|
||
|
|
- if "rule.yml" in filenames and "tests" in dirnames:
|
||
|
|
+ if is_rule_dir(dirpath):
|
||
|
|
short_rule_id = os.path.basename(dirpath)
|
||
|
|
|
||
|
|
# Load the rule itself to check for a template.
|
||
|
|
@@ -543,13 +550,14 @@ def iterate_over_rules(product=None):
|
||
|
|
# like the behavior in template_tests, this will overwrite any
|
||
|
|
# templated tests with the same file name.
|
||
|
|
tests_dir = os.path.join(dirpath, "tests")
|
||
|
|
- tests_dir_files = os.listdir(tests_dir)
|
||
|
|
- for test_case in tests_dir_files:
|
||
|
|
- test_path = os.path.join(tests_dir, test_case)
|
||
|
|
- if os.path.isdir(test_path):
|
||
|
|
- continue
|
||
|
|
+ if os.path.exists(tests_dir):
|
||
|
|
+ tests_dir_files = os.listdir(tests_dir)
|
||
|
|
+ for test_case in tests_dir_files:
|
||
|
|
+ test_path = os.path.join(tests_dir, test_case)
|
||
|
|
+ if os.path.isdir(test_path):
|
||
|
|
+ continue
|
||
|
|
|
||
|
|
- all_tests[test_case] = process_file(test_path, local_env_yaml)
|
||
|
|
+ all_tests[test_case] = process_file(test_path, local_env_yaml)
|
||
|
|
|
||
|
|
# Filter out everything except the shell test scenarios.
|
||
|
|
# Other files in rule directories are editor swap files
|
||
|
|
@@ -557,6 +565,12 @@ def iterate_over_rules(product=None):
|
||
|
|
allowed_scripts = filter(lambda x: x.endswith(".sh"), all_tests)
|
||
|
|
content_mapping = {x: all_tests[x] for x in allowed_scripts}
|
||
|
|
|
||
|
|
+ # Skip any rules that lack any content. This ensures that if we
|
||
|
|
+ # end up with rules with a template lacking tests and without any
|
||
|
|
+ # rule directory tests, we don't include the empty rule here.
|
||
|
|
+ if not content_mapping:
|
||
|
|
+ continue
|
||
|
|
+
|
||
|
|
full_rule_id = OSCAP_RULE + short_rule_id
|
||
|
|
result = Rule(
|
||
|
|
directory=tests_dir, id=full_rule_id, short_id=short_rule_id,
|
||
|
|
|
||
|
|
From 0661e505c1294a7d5bf6a59813afabff48f31b4c Mon Sep 17 00:00:00 2001
|
||
|
|
From: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
Date: Tue, 27 Jul 2021 12:23:25 -0400
|
||
|
|
Subject: [PATCH 17/19] Template tests with matching prodtype
|
||
|
|
|
||
|
|
When prodtype is known to the test system, we can skip templating any
|
||
|
|
rule that has a prodtype that doesn't match. This saves time during
|
||
|
|
building the bundle and restricts us to only writing tests we care
|
||
|
|
about and can potentially use.
|
||
|
|
|
||
|
|
Note that there might be a mismatch between (datastream, rule): if a
|
||
|
|
rule is updated on disk but the datastream isn't regenerated, you might
|
||
|
|
run into weird edge cases where this either over- or under-provisions,
|
||
|
|
but the same issue would likely occur previously.
|
||
|
|
|
||
|
|
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
---
|
||
|
|
tests/ssg_test_suite/common.py | 22 ++++++++++++++++++++++
|
||
|
|
1 file changed, 22 insertions(+)
|
||
|
|
|
||
|
|
diff --git a/tests/ssg_test_suite/common.py b/tests/ssg_test_suite/common.py
|
||
|
|
index 0cb5451e31b..946d7152af1 100644
|
||
|
|
--- a/tests/ssg_test_suite/common.py
|
||
|
|
+++ b/tests/ssg_test_suite/common.py
|
||
|
|
@@ -378,6 +378,17 @@ def template_rule_tests(product, product_yaml, template_builder, tmpdir, dirpath
|
||
|
|
# Load the rule and its environment
|
||
|
|
rule, local_env_yaml = load_rule_and_env(dirpath, product_yaml, product)
|
||
|
|
|
||
|
|
+ # Before we get too far, we wish to search the rule YAML to see if
|
||
|
|
+ # it is applicable to the current product. If we have a product
|
||
|
|
+ # and the rule isn't applicable for the product, there's no point
|
||
|
|
+ # in continuing with the rest of the loading. This should speed up
|
||
|
|
+ # the loading of the templated tests. Note that we've already
|
||
|
|
+ # parsed the prodtype into local_env_yaml
|
||
|
|
+ if product and local_env_yaml['products']:
|
||
|
|
+ prodtypes = local_env_yaml['products']
|
||
|
|
+ if "all" not in prodtypes and product not in prodtypes:
|
||
|
|
+ return
|
||
|
|
+
|
||
|
|
# Create the destination directory.
|
||
|
|
dest_path = os.path.join(tmpdir, rule.id_)
|
||
|
|
os.mkdir(dest_path)
|
||
|
|
@@ -532,6 +543,17 @@ def iterate_over_rules(product=None):
|
||
|
|
# Load the rule itself to check for a template.
|
||
|
|
rule, local_env_yaml = load_rule_and_env(dirpath, product_yaml, product)
|
||
|
|
|
||
|
|
+ # Before we get too far, we wish to search the rule YAML to see if
|
||
|
|
+ # it is applicable to the current product. If we have a product
|
||
|
|
+ # and the rule isn't applicable for the product, there's no point
|
||
|
|
+ # in continuing with the rest of the loading. This should speed up
|
||
|
|
+ # the loading of the templated tests. Note that we've already
|
||
|
|
+ # parsed the prodtype into local_env_yaml
|
||
|
|
+ if product and local_env_yaml['products']:
|
||
|
|
+ prodtypes = local_env_yaml['products']
|
||
|
|
+ if "all" not in prodtypes and product not in prodtypes:
|
||
|
|
+ continue
|
||
|
|
+
|
||
|
|
# All tests is a mapping from path (in the tarball) to contents
|
||
|
|
# of the test case. This is necessary because later code (which
|
||
|
|
# attempts to parse headers from the test case) don't have easy
|
||
|
|
|
||
|
|
From f1c0bbd7fb8af5f0af96bf3d442166724fcc7f94 Mon Sep 17 00:00:00 2001
|
||
|
|
From: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
Date: Tue, 27 Jul 2021 14:51:53 -0400
|
||
|
|
Subject: [PATCH 18/19] Use process_file_with_macros rather than process_file
|
||
|
|
|
||
|
|
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
---
|
||
|
|
tests/ssg_test_suite/common.py | 6 +++---
|
||
|
|
1 file changed, 3 insertions(+), 3 deletions(-)
|
||
|
|
|
||
|
|
diff --git a/tests/ssg_test_suite/common.py b/tests/ssg_test_suite/common.py
|
||
|
|
index 946d7152af1..291e0f5c9ad 100644
|
||
|
|
--- a/tests/ssg_test_suite/common.py
|
||
|
|
+++ b/tests/ssg_test_suite/common.py
|
||
|
|
@@ -15,7 +15,7 @@
|
||
|
|
from ssg.constants import MULTI_PLATFORM_MAPPING
|
||
|
|
from ssg.constants import FULL_NAME_TO_PRODUCT_MAPPING
|
||
|
|
from ssg.constants import OSCAP_RULE
|
||
|
|
-from ssg.jinja import process_file
|
||
|
|
+from ssg.jinja import process_file_with_macros
|
||
|
|
from ssg.products import product_yaml_path, load_product_yaml
|
||
|
|
from ssg.rules import get_rule_dir_yaml, is_rule_dir
|
||
|
|
from ssg.rule_yaml import parse_prodtype
|
||
|
|
@@ -364,7 +364,7 @@ def write_rule_dir_tests(local_env_yaml, dest_path, dirpath):
|
||
|
|
# Rather than performing an OS-level copy, we need to
|
||
|
|
# first parse the test with jinja and then write it back
|
||
|
|
# out to the destination.
|
||
|
|
- parsed_test = process_file(src_test_path, local_env_yaml)
|
||
|
|
+ parsed_test = process_file_with_macros(src_test_path, local_env_yaml)
|
||
|
|
with open(dest_test_path, 'w') as output_fp:
|
||
|
|
print(parsed_test, file=output_fp)
|
||
|
|
|
||
|
|
@@ -579,7 +579,7 @@ def iterate_over_rules(product=None):
|
||
|
|
if os.path.isdir(test_path):
|
||
|
|
continue
|
||
|
|
|
||
|
|
- all_tests[test_case] = process_file(test_path, local_env_yaml)
|
||
|
|
+ all_tests[test_case] = process_file_with_macros(test_path, local_env_yaml)
|
||
|
|
|
||
|
|
# Filter out everything except the shell test scenarios.
|
||
|
|
# Other files in rule directories are editor swap files
|
||
|
|
|
||
|
|
From 0f22db3a49bf8ccb14e0ea52e7884a525fa37f6f Mon Sep 17 00:00:00 2001
|
||
|
|
From: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
Date: Thu, 29 Jul 2021 07:45:41 -0400
|
||
|
|
Subject: [PATCH 19/19] Add an option to not run duplicate templated tests
|
||
|
|
|
||
|
|
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
|
||
|
|
---
|
||
|
|
tests/ssg_test_suite/combined.py | 4 ++--
|
||
|
|
tests/ssg_test_suite/common.py | 9 ++++++---
|
||
|
|
tests/ssg_test_suite/rule.py | 19 +++++++++++++++----
|
||
|
|
tests/test_suite.py | 10 ++++++++++
|
||
|
|
4 files changed, 33 insertions(+), 9 deletions(-)
|
||
|
|
|
||
|
|
diff --git a/tests/ssg_test_suite/combined.py b/tests/ssg_test_suite/combined.py
|
||
|
|
index 05270353235..4ef8898f602 100644
|
||
|
|
--- a/tests/ssg_test_suite/combined.py
|
||
|
|
+++ b/tests/ssg_test_suite/combined.py
|
||
|
|
@@ -39,10 +39,10 @@ def __init__(self, test_env):
|
||
|
|
self.results = list()
|
||
|
|
self._current_result = None
|
||
|
|
|
||
|
|
- def _rule_should_be_tested(self, rule, rules_to_be_tested):
|
||
|
|
+ def _rule_should_be_tested(self, rule, rules_to_be_tested, tested_templates):
|
||
|
|
if rule.short_id not in rules_to_be_tested:
|
||
|
|
return False
|
||
|
|
- return True
|
||
|
|
+ return not self._rule_template_been_tested(rule, tested_templates)
|
||
|
|
|
||
|
|
def _modify_parameters(self, script, params):
|
||
|
|
# If there is no profiles metadata in a script we will use
|
||
|
|
diff --git a/tests/ssg_test_suite/common.py b/tests/ssg_test_suite/common.py
|
||
|
|
index 291e0f5c9ad..132a004323f 100644
|
||
|
|
--- a/tests/ssg_test_suite/common.py
|
||
|
|
+++ b/tests/ssg_test_suite/common.py
|
||
|
|
@@ -31,7 +31,7 @@
|
||
|
|
"Scenario_conditions",
|
||
|
|
("backend", "scanning_mode", "remediated_by", "datastream"))
|
||
|
|
Rule = namedtuple(
|
||
|
|
- "Rule", ["directory", "id", "short_id", "files"])
|
||
|
|
+ "Rule", ["directory", "id", "short_id", "files", "template"])
|
||
|
|
|
||
|
|
SSG_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
|
||
|
|
|
||
|
|
@@ -542,6 +542,7 @@ def iterate_over_rules(product=None):
|
||
|
|
|
||
|
|
# Load the rule itself to check for a template.
|
||
|
|
rule, local_env_yaml = load_rule_and_env(dirpath, product_yaml, product)
|
||
|
|
+ template_name = None
|
||
|
|
|
||
|
|
# Before we get too far, we wish to search the rule YAML to see if
|
||
|
|
# it is applicable to the current product. If we have a product
|
||
|
|
@@ -562,11 +563,13 @@ def iterate_over_rules(product=None):
|
||
|
|
# templating system.
|
||
|
|
all_tests = dict()
|
||
|
|
|
||
|
|
- # Start
|
||
|
|
+ # Start by checking for templating tests and provision them if
|
||
|
|
+ # present.
|
||
|
|
if rule.template and rule.template['vars']:
|
||
|
|
templated_tests = template_builder.get_all_tests(
|
||
|
|
rule.id_, rule.template, local_env_yaml)
|
||
|
|
all_tests.update(templated_tests)
|
||
|
|
+ template_name = rule.template['name']
|
||
|
|
|
||
|
|
# Add additional tests from the local rule directory. Note that,
|
||
|
|
# like the behavior in template_tests, this will overwrite any
|
||
|
|
@@ -596,7 +599,7 @@ def iterate_over_rules(product=None):
|
||
|
|
full_rule_id = OSCAP_RULE + short_rule_id
|
||
|
|
result = Rule(
|
||
|
|
directory=tests_dir, id=full_rule_id, short_id=short_rule_id,
|
||
|
|
- files=content_mapping)
|
||
|
|
+ files=content_mapping, template=template_name)
|
||
|
|
yield result
|
||
|
|
|
||
|
|
|
||
|
|
diff --git a/tests/ssg_test_suite/rule.py b/tests/ssg_test_suite/rule.py
|
||
|
|
index 5ad7eb8ed27..b707326179f 100644
|
||
|
|
--- a/tests/ssg_test_suite/rule.py
|
||
|
|
+++ b/tests/ssg_test_suite/rule.py
|
||
|
|
@@ -211,13 +211,23 @@ def _final_scan_went_ok(self, runner, rule_id):
|
||
|
|
logging.error(msg)
|
||
|
|
return success
|
||
|
|
|
||
|
|
- def _rule_should_be_tested(self, rule, rules_to_be_tested):
|
||
|
|
+ def _rule_template_been_tested(self, rule, tested_templates):
|
||
|
|
+ if rule.template is None:
|
||
|
|
+ return False
|
||
|
|
+ if self.test_env.duplicate_templates:
|
||
|
|
+ return False
|
||
|
|
+ if rule.template in tested_templates:
|
||
|
|
+ return True
|
||
|
|
+ tested_templates.add(rule.template)
|
||
|
|
+ return False
|
||
|
|
+
|
||
|
|
+ def _rule_should_be_tested(self, rule, rules_to_be_tested, tested_templates):
|
||
|
|
if 'ALL' in rules_to_be_tested:
|
||
|
|
# don't select rules that are not present in benchmark
|
||
|
|
if not xml_operations.find_rule_in_benchmark(
|
||
|
|
self.datastream, self.benchmark_id, rule.id):
|
||
|
|
return False
|
||
|
|
- return True
|
||
|
|
+ return not self._rule_template_been_tested(rule, tested_templates)
|
||
|
|
else:
|
||
|
|
for rule_to_be_tested in rules_to_be_tested:
|
||
|
|
# we check for a substring
|
||
|
|
@@ -226,7 +236,7 @@ def _rule_should_be_tested(self, rule, rules_to_be_tested):
|
||
|
|
else:
|
||
|
|
pattern = OSCAP_RULE + rule_to_be_tested
|
||
|
|
if fnmatch.fnmatch(rule.id, pattern):
|
||
|
|
- return True
|
||
|
|
+ return not self._rule_template_been_tested(rule, tested_templates)
|
||
|
|
return False
|
||
|
|
|
||
|
|
def _ensure_package_present_for_all_scenarios(self, scenarios_by_rule):
|
||
|
|
@@ -250,8 +260,9 @@ def _prepare_environment(self, scenarios_by_rule):
|
||
|
|
|
||
|
|
def _get_rules_to_test(self, target):
|
||
|
|
rules_to_test = []
|
||
|
|
+ tested_templates = set()
|
||
|
|
for rule in common.iterate_over_rules(self.test_env.product):
|
||
|
|
- if not self._rule_should_be_tested(rule, target):
|
||
|
|
+ if not self._rule_should_be_tested(rule, target, tested_templates):
|
||
|
|
continue
|
||
|
|
if not xml_operations.find_rule_in_benchmark(
|
||
|
|
self.datastream, self.benchmark_id, rule.id):
|
||
|
|
diff --git a/tests/test_suite.py b/tests/test_suite.py
|
||
|
|
index 00da15329a5..445a53f41d8 100755
|
||
|
|
--- a/tests/test_suite.py
|
||
|
|
+++ b/tests/test_suite.py
|
||
|
|
@@ -116,6 +116,15 @@ def parse_args():
|
||
|
|
"or remediation done by using remediation roles "
|
||
|
|
"that are saved to disk beforehand.")
|
||
|
|
|
||
|
|
+ common_parser.add_argument(
|
||
|
|
+ "--duplicate-templates",
|
||
|
|
+ dest="duplicate_templates",
|
||
|
|
+ default=False,
|
||
|
|
+ action="store_true",
|
||
|
|
+ help="Execute all tests even for tests using shared templates; "
|
||
|
|
+ "otherwise, executes one test per template type"
|
||
|
|
+ )
|
||
|
|
+
|
||
|
|
subparsers = parser.add_subparsers(dest="subparser_name",
|
||
|
|
help="Subcommands: profile, rule, combined")
|
||
|
|
subparsers.required = True
|
||
|
|
@@ -345,6 +354,7 @@ def normalize_passed_arguments(options):
|
||
|
|
# Add in product to the test environment. This is independent of actual
|
||
|
|
# test environment type so we do it after creation.
|
||
|
|
options.test_env.product = options.product
|
||
|
|
+ options.test_env.duplicate_templates = options.duplicate_templates
|
||
|
|
|
||
|
|
try:
|
||
|
|
benchmark_cpes = xml_operations.benchmark_get_applicable_platforms(
|