From b220c2111023f1d1f143c142f8d5a2a06aadd387 Mon Sep 17 00:00:00 2001 From: Noriko Hosoi Date: Tue, 5 Apr 2022 08:49:32 -0700 Subject: [PATCH] RFE - support template, severity and facility options - template options to the files & formats output - severity and facility options to the files input (issue #271) --- README.md | 9 ++++ defaults/main.yml | 8 +++ roles/rsyslog/defaults/main.yml | 6 --- roles/rsyslog/tasks/inputs/files/main.yml | 4 +- roles/rsyslog/templates/input_files.j2 | 12 +++++ roles/rsyslog/templates/output_forwards.j2 | 7 +++ roles/rsyslog/vars/outputs/files/main.yml | 10 ++-- roles/rsyslog/vars/outputs/forwards/main.yml | 26 +++++++++- tests/tests_basics_files.yml | 1 + tests/tests_basics_forwards.yml | 19 +++++++ tests/tests_combination.yml | 38 ++++++++------ tests/tests_files_files.yml | 53 +++++++++++++------- 12 files changed, 149 insertions(+), 44 deletions(-) create mode 100644 roles/rsyslog/templates/input_files.j2 diff --git a/README.md b/README.md index 837e7825..b0a2a91c 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,8 @@ Available options: Available options: - `input_log_path`: File name to be read by the imfile plugin. The value should be full path. Wildcard '\*' is allowed in the path. Default to `/var/log/containers/*.log`. +`facility`: Facility to filter the inputs from the files. +`severity`: Severity to filter the inputs from the files. #### ovirt type @@ -246,6 +248,9 @@ Available options: - `property_value`: Value in property-based filter; default to `error` - `path`: Path to the output file. +logging_files_template_format: Set default template for the files output. +Allowed values are `traditional`, `syslog`, and `modern`. Default to `modern`. + **Note:** Selector options and property-based filter options are exclusive. If Property-based filter options are defined, selector options will be ignored. **Note:** Unless the above options are given, these local file outputs are configured. @@ -278,6 +283,10 @@ Available options: - `tls`: Set to `true` to encrypt the connection using the default TLS implementation used by the provider. Default to `false`. - `pki_authmode`: Specifying the default network driver authentication mode. `x509/name`, `x509/fingerprint`, or `anon` is accepted. Default to `x509/name`. - `permitted_server`: Hostname, IP address, fingerprint(sha1) or wildcard DNS domain of the server which this client will be allowed to connect and send logs over TLS. Default to `*.{{ logging_domain }}` +- `template`: Template format for the particular forwards output. Allowed values are `traditional`, `syslog`, and `modern`. Default to `modern`. + +logging_forwards_template_format: Set default template for the forwards output. +Allowed values are `traditional`, `syslog`, and `modern`. Default to `modern`. **Note:** Selector options and property-based filter options are exclusive. If Property-based filter options are defined, selector options will be ignored. diff --git a/defaults/main.yml b/defaults/main.yml index 7c6ae2df..358036fd 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -81,3 +81,11 @@ logging_domain: '{{ ansible_domain if ansible_domain else ansible_hostname }}' # # Password to pass to the elasticsearch output logging_elasticsearch_password: "" + +# Output file format +# Allowed values: "traditional", "syslog", or "modern"; default to "modern" +logging_files_template_format: "" + +# Output forward format +# Allowed values: "traditional", "syslog", or "modern"; default to "modern" +logging_forwards_template_format: "" diff --git a/roles/rsyslog/defaults/main.yml b/roles/rsyslog/defaults/main.yml index 3b63df9e..f19140bd 100644 --- a/roles/rsyslog/defaults/main.yml +++ b/roles/rsyslog/defaults/main.yml @@ -31,9 +31,3 @@ rsyslog_extra_packages: [] # List of additional custom config files. # Each element: full paths to the files to be deployed. rsyslog_custom_config_files: [] - -# rsyslog_basics_use_traditional_timestamp_format -# -# Traditional timestamp format looks like 'Mar 27 14:16:47' -# By setting false, it'd change 2020-03-27T14:16:47.139796+00:00) -rsyslog_basics_use_traditional_timestamp_format: true diff --git a/roles/rsyslog/tasks/inputs/files/main.yml b/roles/rsyslog/tasks/inputs/files/main.yml index 379be267..b805dea6 100644 --- a/roles/rsyslog/tasks/inputs/files/main.yml +++ b/roles/rsyslog/tasks/inputs/files/main.yml @@ -18,9 +18,7 @@ type: input state: "{{ __rsyslog_input.state | d('present') }}" sections: - - options: |- - input(type="imfile" file="{{ __rsyslog_input.input_log_path }}" tag="{{ __rsyslog_input.name }}") - {{ lookup("template", "input_template.j2") }} + - options: "{{ lookup('template', 'input_files.j2') }}" include_tasks: file: "{{ role_path }}/tasks/deploy.yml" when: diff --git a/roles/rsyslog/templates/input_files.j2 b/roles/rsyslog/templates/input_files.j2 new file mode 100644 index 00000000..9b317b06 --- /dev/null +++ b/roles/rsyslog/templates/input_files.j2 @@ -0,0 +1,12 @@ +input( + type="imfile" + file="{{ __rsyslog_input.input_log_path }}" + tag="{{ __rsyslog_input.name }}" +{% if __rsyslog_input.severity is defined %} + severity="{{ __rsyslog_input.severity }}" +{% endif %} +{% if __rsyslog_input.facility is defined %} + facility="{{ __rsyslog_input.facility }}" +{% endif %} +) +{{ lookup('template', 'input_template.j2') }} diff --git a/roles/rsyslog/templates/output_forwards.j2 b/roles/rsyslog/templates/output_forwards.j2 index afa40e38..2048b3b6 100644 --- a/roles/rsyslog/templates/output_forwards.j2 +++ b/roles/rsyslog/templates/output_forwards.j2 @@ -29,6 +29,13 @@ ruleset(name="{{ __rsyslog_output.name }}") { {% endif %} {% if __forwards_protocol != '' %} Protocol="{{ __forwards_protocol }}" +{% endif %} +{% if __rsyslog_output.template | d('') == 'traditional' %} + Template="RSYSLOG_TraditionalForwardFormat" +{% elif __rsyslog_output.template | d('') == 'syslog' %} + Template="RSYSLOG_SyslogProtocol23Format" +{% else %} + Template="RSYSLOG_ForwardFormat" {% endif %} ) } diff --git a/roles/rsyslog/vars/outputs/files/main.yml b/roles/rsyslog/vars/outputs/files/main.yml index 06dacc66..a64a6452 100644 --- a/roles/rsyslog/vars/outputs/files/main.yml +++ b/roles/rsyslog/vars/outputs/files/main.yml @@ -19,11 +19,15 @@ __rsyslog_conf_files_output_modules: type: 'modules' sections: - - comment: 'Log messages into files with timestamp format either - traditional or default' + # yamllint disable rule:line-length + - comment: 'Log messages into files with traditional, syslog, or + default format' options: |- - {% if rsyslog_basics_use_traditional_timestamp_format | d(true) %} + {% if logging_files_template_format == "traditional" %} module(load="builtin:omfile" Template="RSYSLOG_TraditionalFileFormat") + {% elif logging_files_template_format == "syslog" %} + module(load="builtin:omfile" Template="RSYSLOG_SyslogProtocol23Format") {% else %} module(load="builtin:omfile") {% endif %} + # yamllint enable rule:line-length diff --git a/roles/rsyslog/vars/outputs/forwards/main.yml b/roles/rsyslog/vars/outputs/forwards/main.yml index f2041c8f..194c9f8d 100644 --- a/roles/rsyslog/vars/outputs/forwards/main.yml +++ b/roles/rsyslog/vars/outputs/forwards/main.yml @@ -6,4 +6,28 @@ # List of rpm packages for Forwards output. __rsyslog_forwards_output_packages: [] -__rsyslog_forwards_output_rules: [] + +# Forwards Rsyslog output configuration rules +# ------------------------------------------- +__rsyslog_forwards_output_rules: + - '{{ __rsyslog_conf_forwards_output_modules }}' + +# __rsyslog_conf_forwards_output_modules: +__rsyslog_conf_forwards_output_modules: + + - name: 'output-forwards-modules' + type: 'modules' + sections: + + # yamllint disable rule:line-length + - comment: 'Log messages into files with traditional, syslog, or + default format' + options: |- + {% if logging_forwards_template_format == "traditional" %} + module(load="builtin:omfwd" Template="RSYSLOG_TraditionalForwardFormat") + {% elif logging_forwards_template_format == "syslog" %} + module(load="builtin:omfwd" Template="RSYSLOG_SyslogProtocol23Format") + {% else %} + module(load="builtin:omfwd") + {% endif %} + # yamllint enable rule:line-length diff --git a/tests/tests_basics_files.yml b/tests/tests_basics_files.yml index 2955cbd4..ec585978 100644 --- a/tests/tests_basics_files.yml +++ b/tests/tests_basics_files.yml @@ -324,6 +324,7 @@ Target="host.domain" Port="1514" Protocol="tcp" + Template="RSYSLOG_ForwardFormat" ) } mode: '0600' diff --git a/tests/tests_basics_forwards.yml b/tests/tests_basics_forwards.yml index 5a2bfc71..9d629774 100644 --- a/tests/tests_basics_forwards.yml +++ b/tests/tests_basics_forwards.yml @@ -5,6 +5,7 @@ vars: __test_files_conf: >- /etc/rsyslog.d/30-output-files-default_files.conf + __test_forward_module_conf: /etc/rsyslog.d/10-output-forwards-modules.conf __test_forward_conf_s_f: >- /etc/rsyslog.d/30-output-forwards-forwards_severity_and_facility.conf __test_forward_conf_f: >- @@ -27,12 +28,16 @@ __expected_error: "Error: tls is enabled in forwards_severity_and_facility; you must specify logging_pki_files ca_cert_src and/or ca_cert in the playbook var section." + __test_template: RSYSLOG_ForwardFormat + __test_template_trad: RSYSLOG_TraditionalForwardFormat + __test_template_sys: RSYSLOG_SyslogProtocol23Format tasks: # TEST CASE 0 - name: "TEST CASE 0; Ensure that the logs from basics inputs are sent to the forwards outputs and implicit files output" vars: + logging_forwards_template_format: traditional logging_outputs: - name: forwards_severity_and_facility type: forwards @@ -40,6 +45,7 @@ severity: info target: host.domain tcp_port: 1514 + template: syslog - name: forwards_facility_only type: forwards facility: local2 @@ -112,6 +118,11 @@ - ca-certificates include_tasks: tasks/check_packages.yml + - name: >- + Check the module param template is set to "{{ __test_template_trad }}" + command: >- + grep '{{ __test_template_trad }}' '{{ __test_forward_module_conf }}' + - name: Generate a file to check severity_and_facility copy: dest: /tmp/__testfile__ @@ -125,6 +136,7 @@ Target="host.domain" Port="1514" Protocol="tcp" + Template="{{ __test_template_sys }}" ) } mode: '0600' @@ -146,6 +158,7 @@ Target="host.domain" Port="2514" Protocol="tcp" + Template="{{ __test_template }}" ) } mode: '0600' @@ -167,6 +180,7 @@ Target="host.domain" Port="3514" Protocol="tcp" + Template="{{ __test_template }}" ) } mode: '0600' @@ -188,6 +202,7 @@ Target="host.domain" Port="4514" Protocol="tcp" + Template="{{ __test_template }}" ) } mode: '0600' @@ -209,6 +224,7 @@ Target="host.domain" Port="6514" Protocol="udp" + Template="{{ __test_template }}" ) } mode: '0600' @@ -228,6 +244,7 @@ *.* action(name="forwards_no_severity_and_facility_protocol_port" type="omfwd" Target="host.domain" + Template="{{ __test_template }}" ) } mode: '0600' @@ -356,6 +373,7 @@ StreamDriverPermittedPeers="*.example.com" Port="1514" Protocol="tcp" + Template="{{ __test_template }}" ) } mode: '0600' @@ -479,6 +497,7 @@ StreamDriverPermittedPeers="*.example.com" Port="1514" Protocol="tcp" + Template="{{ __test_template }}" ) } mode: '0600' diff --git a/tests/tests_combination.yml b/tests/tests_combination.yml index b53253e5..7ac9918f 100644 --- a/tests/tests_combination.yml +++ b/tests/tests_combination.yml @@ -130,6 +130,7 @@ Target="host.domain" Port="1514" Protocol="tcp" + Template="RSYSLOG_ForwardFormat" ) } mode: '0600' @@ -145,13 +146,17 @@ failed_when: not __result.stat.exists - name: Check the input call with tag={{ __test_tag }} - command: >- - /bin/grep - ' *input(type="imfile" - file="/var/log/inputdirectory/\*.log" - tag="{{ __test_tag }}"' - {{ __test_inputfiles_conf }} - changed_when: false + lineinfile: + path: "{{ __test_inputfiles_conf }}" + line: " {{ item }}" + state: present + check_mode: yes + register: _result + failed_when: _result.changed + loop: + - type="imfile" + - file="/var/log/inputdirectory/*.log" + - tag="{{ __test_tag }}" # yamllint disable rule:line-length - name: "Create a test log file with a log message in @@ -295,6 +300,7 @@ Target="host.domain" Port="1514" Protocol="tcp" + Template="RSYSLOG_ForwardFormat" ) } mode: '0600' @@ -310,13 +316,17 @@ failed_when: not __result.stat.exists - name: Check the input call with tag={{ __test_tag }} - command: >- - /bin/grep - ' *input(type="imfile" - file="/var/log/inputdirectory/\*.log" - tag="{{ __test_tag }}"' - {{ __test_inputfiles_conf }} - changed_when: false + lineinfile: + path: "{{ __test_inputfiles_conf }}" + line: " {{ item }}" + state: present + check_mode: yes + register: _result + failed_when: _result.changed + loop: + - type="imfile" + - file="/var/log/inputdirectory/*.log" + - tag="{{ __test_tag }}" # yamllint disable rule:line-length - name: "Create a test log file with a log message in diff --git a/tests/tests_files_files.yml b/tests/tests_files_files.yml index 5b5a8b0e..a92be260 100644 --- a/tests/tests_files_files.yml +++ b/tests/tests_files_files.yml @@ -31,16 +31,19 @@ __test_inputfiles_dir0: /var/log/inputdirectory0 __test_inputfiles_dir1: /var/log/inputdirectory1 __test_inputfiles_conf0: /etc/rsyslog.d/90-input-files-files_input0.conf - __test_inputfiles_conf1: /etc/rsyslog.d/90-input-files-files_input1.conf + __test_inputfiles_conf1: /etc/rsyslog.d/90-input-files-php:.conf __test_inputfiles_conf2: /etc/rsyslog.d/90-input-files-files_input2.conf __test_outputfiles_conf: /etc/rsyslog.d/30-output-files-files_output0.conf __default_system_log: /var/log/messages + __test_outputfiles_module_conf: /etc/rsyslog.d/10-output-files-modules.conf + __test_template: RSYSLOG_TraditionalFileFormat tasks: # TEST CASE 0 - name: "TEST CASE 0; Ensure that the role runs with parameters from two files inputs to two files outputs" vars: + logging_files_template_format: traditional logging_outputs: - name: files_output0 type: files @@ -59,14 +62,16 @@ - name: files_input0 type: files input_log_path: "{{ __test_inputfiles_dir0 }}/*.log" - - name: files_input1 + severity: err + facility: local6 + - name: "php:" type: files input_log_path: "{{ __test_inputfiles_dir1 }}/*.log" - name: files_input2 type: files logging_flows: - name: flow_0 - inputs: [files_input0, files_input1] + inputs: [files_input0, "php:"] outputs: [files_output0, files_output1] include_role: name: linux-system-roles.logging @@ -95,6 +100,10 @@ - ca-certificates include_tasks: tasks/check_packages.yml + - name: "Check the module param template is set to {{ __test_template }}" + command: >- + grep '{{ __test_template }}' '{{ __test_outputfiles_module_conf }}' + - name: Check if the input files config does not exist stat: path: "{{ __test_inputfiles_conf2 }}" @@ -119,22 +128,32 @@ failed_when: not __result.stat.exists - name: Check the filter 0 - command: >- - /bin/grep - 'input(type="imfile" - file="/var/log/inputdirectory0/\*.log" - tag="files_input0")' - {{ __test_inputfiles_conf0 }} - changed_when: false + lineinfile: + path: "{{ __test_inputfiles_conf0 }}" + line: " {{ item }}" + state: present + check_mode: yes + register: _result + failed_when: _result.changed + loop: + - type="imfile" + - file="/var/log/inputdirectory0/*.log" + - tag="files_input0" + - severity="err" + - facility="local6" - name: Check the filter 1 - command: >- - /bin/grep - 'input(type="imfile" - file="/var/log/inputdirectory1/\*.log" - tag="files_input1")' - {{ __test_inputfiles_conf1 }} - changed_when: false + lineinfile: + path: "{{ __test_inputfiles_conf1 }}" + line: " {{ item }}" + state: present + check_mode: yes + register: _result + failed_when: _result.changed + loop: + - type="imfile" + - file="/var/log/inputdirectory1/*.log" + - tag="php:" - name: END TEST CASE 0; Clean up the deployed config vars: