rhel-system-roles/SOURCES/logging-0005-property-based-filters.diff

From b72e8a48be07a1cebce8b2237d7344220678c2ec Mon Sep 17 00:00:00 2001
From: Noriko Hosoi <nhosoi@redhat.com>
Date: Fri, 16 Oct 2020 08:15:11 -0700
Subject: [PATCH 5/7] Logging - support property-based filters in the files and
 forwards outputs

Adding property-based filter options to files, forwards and remote_files output.
A test case is added to tests_basics_files2.yml.

In addition, fixing a bug caused by a left over file from the previous tests.

Issue - https://github.com/linux-system-roles/logging/issues/179

(cherry picked from commit 6ac8f9ff680a4b0230446062f5927f5921829f80)
---
 README.md                                     | 68 ++++++++++++-------
 roles/rsyslog/templates/output_files.j2       |  4 +-
 roles/rsyslog/templates/output_forwards.j2    |  4 +-
 .../rsyslog/templates/output_remote_files.j2  |  4 +-
 tests/tests_basics_files2.yml                 | 40 +++++++++--
 tests/tests_basics_forwards_cert.yml          |  8 +++
 tests/tests_basics_forwards_cert_missing.yml  |  4 ++
 tests/tests_server_conflict.yml               |  8 +++
 8 files changed, 108 insertions(+), 32 deletions(-)

diff --git a/README.md b/README.md
index db29dc5..4352ee7 100644
--- a/README.md
+++ b/README.md
@@ -180,11 +180,16 @@ This is a schematic logging configuration to show log messages from input_nameA
 
 - `files` type - `files` output supports storing logs in the local files usually in /var/log.<br>
   **available options**
-  - `facility`: Facility; default to `*`.
-  - `severity`: Severity; default to `*`.
-  - `exclude`: Exclude list; default to none.
+  - `facility`: Facility in selector; default to `*`.
+  - `severity`: Severity in selector; default to `*`.
+  - `exclude`: Exclude list used in selector; default to none.
+  - `property`: Property in property-based filter; no default
+  - `prop_op`: Operation in property-based filter; In case of not `!`, put the `prop_op` value in quotes; default to `contains`
+  - `prop_value`: Value in property-based filter; default to `error`
   - `path`: Path to the output file.
 
+  Selector options and property-based filter options are exclusive. If Property-based filter options are defined, selector options will be ignored.
+
   Unless the above options are given, these local file outputs are configured.
   ```
   kern.*                                      /dev/console
@@ -199,8 +204,12 @@ This is a schematic logging configuration to show log messages from input_nameA
 
 - `forwards` type - `forwards` output sends logs to the remote logging system over the network. This is for the client rsyslog.<br>
   **available options**
-  - `facility`: Facility; default to `*`.
-  - `severity`: Severity; default to `*`.
+  - `facility`: Facility in selector; default to `*`.
+  - `severity`: Severity in selector; default to `*`.
+  - `exclude`: Exclude list used in selector; default to none.
+  - `property`: Property in property-based filter; no default
+  - `prop_op`: Operation in property-based filter; In case of not `!`, put the `prop_op` value in quotes; default to `contains`
+  - `prop_value`: Value in property-based filter; default to `error`
   - `target`: Target host (fqdn). **Required**.
   - `udp_port`: UDP port number. Default to `514`.
   - `tcp_port`: TCP port number. Default to `514`.
@@ -208,11 +217,16 @@ This is a schematic logging configuration to show log messages from input_nameA
   - `pki_authmode`: Specifying the default network driver authentication mode. `x509/name`, `x509/fingerprint`, `anon` is accepted. Default to `x509/name`.
   - `permitted_server`: Hostname, IP address, fingerprint(sha1) or wildcard DNS domain of the server which this client will be allowed to connect and send logs over TLS. Default to `*.{{ logging_domain }}`
 
+  Selector options and property-based filter options are exclusive. If Property-based filter options are defined, selector options will be ignored.
+
 - `remote_files` type - `remote_files` output stores logs to the local files per remote host and program name originated the logs.<br>
   **available options**
-  - `facility`: Facility; default to `*`.
-  - `severity`: Severity; default to `*`.
-  - `exclude`: Exclude list; default to none.
+  - `facility`: Facility in selector; default to `*`.
+  - `severity`: Severity in selector; default to `*`.
+  - `exclude`: Exclude list used in selector; default to none.
+  - `property`: Property in property-based filter; no default
+  - `prop_op`: Operation in property-based filter; In case of not `!`, put the `prop_op` value in quotes; default to `contains`
+  - `prop_value`: Value in property-based filter; default to `error`
   - `async_writing`: If set to `true`, the files are written asynchronously. Allowed value is `true` or `false`. Default to `false`.
   - `client_count`: Count of client logging system supported this rsyslog server. Default to `10`.
   - `io_buffer_size`: Buffer size used to write output data. Default to `65536` bytes.
@@ -221,6 +235,8 @@ This is a schematic logging configuration to show log messages from input_nameA
                        `/path/to/output/dir/%HOSTNAME%/%PROGRAMNAME:::secpath-replace%.log`
   - `remote_sub_path`: Relative path to logging_system_log_dir to store the filtered logs.
 
+  Selector options and property-based filter options are exclusive. If Property-based filter options are defined, selector options will be ignored.
+
   if both `remote_log_path` and `remote_sub_path` are _not_ specified, the remote_file output configured with the following settings.
   ```
   template(
@@ -446,32 +462,38 @@ The following playbook generates the same logging configuration files.
         outputs: [files_output0, files_output1]
 ```
 
-5. Deploying `files input` reading logs from a local file and `elasticsearch output` to store the logs. Assuming the ca_cert, cert and key to connect to Elasticsearch are prepared.
+5. Deploying `files input` reading logs from local files and `files output` to write to the local files based on the property-based filters.
 ```yaml
 ---
-- name: Deploying basic input and elasticsearch output
+- name: Deploying files input and configured files output
   hosts: all
   roles:
     - linux-system-roles.logging
   vars:
     logging_inputs:
-      - name: files_input
+      - name: files_input0
         type: files
-        input_log_path: /var/log/containers/*.log
+        input_log_path: /var/log/containerA/*.log
+      - name: files_input1
+        type: files
+        input_log_path: /var/log/containerB/*.log
     logging_outputs:
-      - name: elasticsearch_output
-        type: elasticsearch
-        server_host: your_target_host
-        server_port: 9200
-        index_prefix: project.
-        input_type: ovirt
-        ca_cert_src: /local/path/to/ca_cert
-        cert_src: /local/path/to/cert
-        private_key_src: /local/path/to/key
+      - name: files_output0
+        type: files
+        property: msg
+        prop_op: contains
+        prop_value: error
+        path: /var/log/errors.log
+      - name: files_output1
+        type: files
+        property: msg
+        prop_op: "!contains"
+        prop_value: error
+        path: /var/log/others.log
     logging_flows:
       - name: flow0
-        inputs: [files_input]
-        outputs: [elasticsearch_output]
+        inputs: [files_input0, files_input1]
+        outputs: [files_output0, files_output1]
 ```
 
 ### Client configuration
diff --git a/roles/rsyslog/templates/output_files.j2 b/roles/rsyslog/templates/output_files.j2
index d994414..e15e4cd 100644
--- a/roles/rsyslog/templates/output_files.j2
+++ b/roles/rsyslog/templates/output_files.j2
@@ -1,6 +1,8 @@
 {% if item.path is defined %}
 ruleset(name="{{ item.name }}") {
-{%   if item.exclude | d([]) %}
+{%   if item.property | d() %}
+    :{{ item.property }}, {{ item.prop_op | d('contains') }}, "{{ item.prop_value | d('error') }}" {{ item.path }}
+{%   elif item.exclude | d([]) %}
     {{ item.facility | d('*') }}.{{ item.severity | d('*') }};{{ item.exclude | join(';') }} {{ item.path }}
 {%   else %}
     {{ item.facility | d('*') }}.{{ item.severity | d('*') }} {{ item.path }}
diff --git a/roles/rsyslog/templates/output_forwards.j2 b/roles/rsyslog/templates/output_forwards.j2
index 61254ee..35030b4 100644
--- a/roles/rsyslog/templates/output_forwards.j2
+++ b/roles/rsyslog/templates/output_forwards.j2
@@ -9,7 +9,9 @@
 {%   set __forwards_protocol = '' %}
 {% endif %}
 ruleset(name="{{ item.name }}") {
-{% if item.exclude | d([]) %}
+{% if item.property | d() %}
+    :{{ item.property }}, {{ item.prop_op | d('contains') }}, "{{ item.prop_value | d('error') }}" action(name="{{ item.name }}"
+{% elif item.exclude | d([]) %}
     {{ item.facility | d('*') }}.{{ item.severity | d('*') }};{{ item.exclude | join(';') }} action(name="{{ item.name }}"
 {% else %}
     {{ item.facility | d('*') }}.{{ item.severity | d('*') }} action(name="{{ item.name }}"
diff --git a/roles/rsyslog/templates/output_remote_files.j2 b/roles/rsyslog/templates/output_remote_files.j2
index 3c9339f..aaf547e 100644
--- a/roles/rsyslog/templates/output_remote_files.j2
+++ b/roles/rsyslog/templates/output_remote_files.j2
@@ -17,7 +17,9 @@ ruleset(name="{{ item.name }}"
         queue.size="{{ logging_server_queue_size }}"
         queue.workerThreads="{{ logging_server_threads }}") {
     # Store remote logs in separate logfiles
-{%   if item.exclude | d([]) %}
+{%   if item.property | d() %}
+    :{{ item.property }}, {{ item.prop_op | d('contains') }}, "{{ item.prop_value | d('error') }}" action(name="{{ item.name }}" type="omfile" DynaFile="{{ item.name }}_template" DynaFileCacheSize="{{ item.client_count | d(10) }}" ioBufferSize="{{ item.io_buffer_size | d('65536') }}" asyncWriting="{{ 'on' if item.async_writing | d(false) | bool else 'off' }}")
+{%   elif item.exclude | d([]) %}
     {{ item.facility | d('*') }}.{{ item.severity | d('*') }};{{ item.exclude | join(';') }} action(name="{{ item.name }}" type="omfile" DynaFile="{{ item.name }}_template" DynaFileCacheSize="{{ item.client_count | d(10) }}" ioBufferSize="{{ item.io_buffer_size | d('65536') }}" asyncWriting="{{ 'on' if item.async_writing | d(false) | bool else 'off' }}")
 {%   else %}
     {{ item.facility | d('*') }}.{{ item.severity | d('*') }} action(name="{{ item.name }}" type="omfile" DynaFile="{{ item.name }}_template" DynaFileCacheSize="{{ item.client_count | d(10) }}" ioBufferSize="{{ item.io_buffer_size | d('65536') }}" asyncWriting="{{ 'on' if item.async_writing | d(false) | bool else 'off' }}")
diff --git a/tests/tests_basics_files2.yml b/tests/tests_basics_files2.yml
index 094b125..b1a0f62 100644
--- a/tests/tests_basics_files2.yml
+++ b/tests/tests_basics_files2.yml
@@ -10,9 +10,9 @@
 #    If logging role is executed, the file size is about 100 bytes.
 #    Thus, assert the size is less than 1000.
 # 2. Check file count in /etc/rsyslog.d.
-#    If logging role is executed, 8 config files are generated.
+#    If logging role is executed, 9 config files are generated.
 #    By setting logging_purge_confs, pre-existing config files are deleted.
-#    Thus, assert the the count is equal to 8.
+#    Thus, assert the the count is equal to 9.
 # 3. Check systemctl status of rsyslog as well as error or specific message in the output.
 # 4. To verify the generated filename is correct, check the config file of files output exists.
 # 4.1 Check the config file contains the expected filter and the output file as configured.
@@ -24,6 +24,8 @@
   vars:
     __test_files_conf: /etc/rsyslog.d/30-output-files-files_output1.conf
     __default_system_log: /var/log/messages
+    __prop_based_log0: /var/log/property_based_filter_in.log
+    __prop_based_log1: /var/log/property_based_filter_out.log
 
   tasks:
     - name: deploy config to output into local files
@@ -49,15 +51,23 @@
             path: :omusrmsg:*
           - name: files_output3
             type: files
-            facility: local7
-            path: /var/log/boot.log
+            property: msg
+            prop_op: contains
+            prop_value: property_based_filter_test
+            path: "{{ __prop_based_log0 }}"
+          - name: files_output4
+            type: files
+            property: msg
+            prop_op: "!contains"
+            prop_value: property_based_filter_test
+            path: "{{ __prop_based_log1 }}"
         logging_inputs:
           - name: basic_input
             type: basics
         logging_flows:
           - name: flow_0
             inputs: [basic_input]
-            outputs: [files_output0, files_output1, files_output2, files_output3]
+            outputs: [files_output0, files_output1, files_output2, files_output3, files_output4]
       include_role:
         name: linux-system-roles.logging
 
@@ -74,7 +84,7 @@
 
     - name: Check file counts in rsyslog.d
       assert:
-        that: rsyslog_d_file_count.matched == 8
+        that: rsyslog_d_file_count.matched == 9
 
     # Checking 'error' in stdout from systemctl status is for detecting the case in which rsyslog is running,
     # but some functionality is disabled due to some error, e.g., error: 'tls.cacert' file couldn't be accessed.
@@ -104,3 +114,21 @@
       retries: 5
       delay: 1
       changed_when: false
+
+    - name: Run logger to generate a test log message containing property_based_filter_test
+      command: /bin/logger -i -p local6.info -t testTag1 property_based_filter_test
+      changed_when: false
+
+    - name: Check the test log message in {{ __prop_based_log0 }}
+      command: /bin/grep property_based_filter_test "{{ __prop_based_log0 }}"
+      register: __result
+      until: __result is success
+      retries: 5
+      delay: 1
+      changed_when: false
+
+    - name: Check the test log message not in {{ __prop_based_log1 }}
+      command: /bin/grep property_based_filter_test "{{ __prop_based_log1 }}"
+      register: __result
+      changed_when: false
+      failed_when: "__result is not failed"
diff --git a/tests/tests_basics_forwards_cert.yml b/tests/tests_basics_forwards_cert.yml
index e27e016..48263ae 100644
--- a/tests/tests_basics_forwards_cert.yml
+++ b/tests/tests_basics_forwards_cert.yml
@@ -139,3 +139,11 @@
         - /etc/pki/tls/certs/{{ __test_ca_cert_name }}
         - /etc/pki/tls/certs/{{ __test_cert_name }}
         - /etc/pki/tls/private/{{ __test_key_name }}
+
+    - name: clean up test files
+      file: path="{{ item }}" state=absent
+      loop:
+        - "{{ __test_ca_cert }}"
+        - "{{ __test_cert }}"
+        - "{{ __test_key }}"
+      delegate_to: localhost
diff --git a/tests/tests_basics_forwards_cert_missing.yml b/tests/tests_basics_forwards_cert_missing.yml
index 3e82856..0ad0569 100644
--- a/tests/tests_basics_forwards_cert_missing.yml
+++ b/tests/tests_basics_forwards_cert_missing.yml
@@ -63,6 +63,10 @@
           assert:
             that: "'{{ ansible_failed_result.results.0.msg }}' is match('{{ __expected_error }}')"
 
+    - name: clean up test files
+      file: path="{{ __test_key }}" state=absent
+      delegate_to: localhost
+
     - name: default run for cleanup
       vars:
         logging_inputs:
diff --git a/tests/tests_server_conflict.yml b/tests/tests_server_conflict.yml
index 36eeeb7..8c182f6 100644
--- a/tests/tests_server_conflict.yml
+++ b/tests/tests_server_conflict.yml
@@ -76,3 +76,11 @@
         - assert:
             that: item.msg is not defined or item.msg is defined and item.msg == __expected_error
           loop: "{{ ansible_failed_result.results }}"
+
+    - name: clean up test files
+      file: path="{{ item }}" state=absent
+      loop:
+        - "{{ __test_ca_cert }}"
+        - "{{ __test_cert }}"
+        - "{{ __test_key }}"
+      delegate_to: localhost
-- 
2.26.2