From 2186ad96d1205b68beab70eab69989a690d37133 Mon Sep 17 00:00:00 2001 From: Sergei Petrosian Date: Mon, 27 May 2024 20:31:55 +0200 Subject: [PATCH 1/7] Add check for "rsyslogd: error" in /var/log/messages in all tests /var/log/messages sometimes contains errors not visible in journalctl Tests identified that some of our tests were resulting such errors against certain platforms --- tests/tasks/assert_varlogmessages.yml | 17 +++++++++++++++++ tests/tests_basics_files.yml | 3 +++ tests/tests_basics_forwards.yml | 3 +++ tests/tests_combination.yml | 3 +++ tests/tests_files_elasticsearch.yml | 4 +++- tests/tests_files_files.yml | 3 +++ tests/tests_imuxsock_files.yml | 16 ++-------------- tests/tests_ovirt_elasticsearch.yml | 5 ++++- tests/tests_purge_reset.yml | 4 ++++ tests/tests_relp.yml | 5 ++++- tests/tests_remote.yml | 5 ++++- tests/tests_server.yml | 5 ++++- 12 files changed, 54 insertions(+), 19 deletions(-) create mode 100644 tests/tasks/assert_varlogmessages.yml diff --git a/tests/tasks/assert_varlogmessages.yml b/tests/tasks/assert_varlogmessages.yml new file mode 100644 index 00000000..7cc26eb9 --- /dev/null +++ b/tests/tasks/assert_varlogmessages.yml @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: MIT +--- +- name: Get content of {{ __default_system_log }} + command: cat {{ __default_system_log }} + register: __default_system_log_content + changed_when: false + no_log: true + +# /var/log/messages sometimes contains errors not visible in journalctl +- name: Ensure no errors in {{ __default_system_log }} + assert: + that: "'rsyslogd: error' not in __default_system_log_content.stdout" + +- name: Remove {{ __default_system_log }} + file: + path: "{{ __default_system_log }}" + state: absent diff --git a/tests/tests_basics_files.yml b/tests/tests_basics_files.yml index 0bbc346f..ba60e1b4 100644 --- a/tests/tests_basics_files.yml +++ b/tests/tests_basics_files.yml @@ -392,3 +392,6 @@ file: path: /tmp/__testfile__ state: absent + + - name: Assert {{ __default_system_log }} + include_tasks: tasks/assert_varlogmessages.yml diff --git a/tests/tests_basics_forwards.yml b/tests/tests_basics_forwards.yml index a37b24a5..0372c573 100644 --- a/tests/tests_basics_forwards.yml +++ b/tests/tests_basics_forwards.yml @@ -582,6 +582,9 @@ rmdir /tmp/rsyslog.d-backup changed_when: false + - name: Assert {{ __default_system_log }} + include_tasks: tasks/assert_varlogmessages.yml + # TEST CASE 3 - name: Test case 3 block: diff --git a/tests/tests_combination.yml b/tests/tests_combination.yml index 3c5bc998..6a341a6d 100644 --- a/tests/tests_combination.yml +++ b/tests/tests_combination.yml @@ -577,3 +577,6 @@ - name: "Force all notified handlers to run at this point, not waiting for normal sync points" meta: flush_handlers + + - name: Assert {{ __default_system_log }} + include_tasks: tasks/assert_varlogmessages.yml diff --git a/tests/tests_files_elasticsearch.yml b/tests/tests_files_elasticsearch.yml index 82657043..596fdb71 100644 --- a/tests/tests_files_elasticsearch.yml +++ b/tests/tests_files_elasticsearch.yml @@ -21,7 +21,7 @@ false in the configuration named elasticsearch_output" __certdir: /etc/pki/tls/certs/ __keydir: /etc/pki/tls/private/ - + __default_system_log: /var/log/messages tasks: - name: Run test block: @@ -458,6 +458,8 @@ not waiting for normal sync points" meta: flush_handlers + - name: Assert {{ __default_system_log }} + include_tasks: tasks/assert_varlogmessages.yml always: - name: Remove tempdir file: diff --git a/tests/tests_files_files.yml b/tests/tests_files_files.yml index 047aa82a..90f34247 100644 --- a/tests/tests_files_files.yml +++ b/tests/tests_files_files.yml @@ -172,3 +172,6 @@ - name: Check ports managed by firewall and selinux include_tasks: tasks/check_firewall_selinux.yml + + - name: Assert {{ __default_system_log }} + include_tasks: tasks/assert_varlogmessages.yml diff --git a/tests/tests_imuxsock_files.yml b/tests/tests_imuxsock_files.yml index 9f85b124..86e5f26a 100644 --- a/tests/tests_imuxsock_files.yml +++ b/tests/tests_imuxsock_files.yml @@ -74,14 +74,8 @@ __logging_file: "{{ __default_system_log }}" include_tasks: tasks/test_logger.yml - - name: Get content of {{ __default_system_log }} - command: cat {{ __default_system_log }} - register: __default_system_log_content - changed_when: false - - - name: Ensure no errors in {{ __default_system_log }} - assert: - that: "'rsyslogd: error' not in __default_system_log_content.stdout" + - name: Assert {{ __default_system_log }} + include_tasks: tasks/assert_varlogmessages.yml - name: Check ports managed by firewall and selinux include_tasks: tasks/check_firewall_selinux.yml @@ -148,12 +142,6 @@ command: journalctl -ex changed_when: false - # When imuxsock is configured, errors are not visible in journalctl - - name: Print errors in {{ __default_system_log }} - command: >- - grep "rsyslogd: error" {{ __default_system_log }} - changed_when: false - - name: Fail fail: msg: "{{ ansible_failed_result }}" diff --git a/tests/tests_ovirt_elasticsearch.yml b/tests/tests_ovirt_elasticsearch.yml index 706ac278..45df747d 100644 --- a/tests/tests_ovirt_elasticsearch.yml +++ b/tests/tests_ovirt_elasticsearch.yml @@ -24,7 +24,7 @@ __test_logs_index: project.ovirt-logs __test_engine_input: /var/log/ovirt-engine/engine.log __test_vdsm_input: /var/log/vdsm/vdsm.log - + __default_system_log: /var/log/messages tasks: - name: TEST CASE 0; Ensure basic ovirt default configuration works vars: @@ -376,3 +376,6 @@ - name: "Force all notified handlers to run at this point, not waiting for normal sync points" meta: flush_handlers + + - name: Assert {{ __default_system_log }} + include_tasks: tasks/assert_varlogmessages.yml diff --git a/tests/tests_purge_reset.yml b/tests/tests_purge_reset.yml index 051a788c..6eb41593 100644 --- a/tests/tests_purge_reset.yml +++ b/tests/tests_purge_reset.yml @@ -6,6 +6,7 @@ hosts: all vars: __test_default_files_conf: /etc/rsyslog.d/30-output-files-default_files.conf + __default_system_log: /var/log/messages tasks: - name: Determine if system is ostree and set flag when: not __logging_is_ostree is defined @@ -151,3 +152,6 @@ - name: Check ports managed by firewall and selinux include_tasks: tasks/check_firewall_selinux.yml + + - name: Assert {{ __default_system_log }} + include_tasks: tasks/assert_varlogmessages.yml diff --git a/tests/tests_relp.yml b/tests/tests_relp.yml index d04dbadf..1d36bbfa 100644 --- a/tests/tests_relp.yml +++ b/tests/tests_relp.yml @@ -15,7 +15,7 @@ __test_relp_global: /etc/rsyslog.d/00-global.conf logging_max_message_size: 16384 logging_preserve_fqdn: true - + __default_system_log: /var/log/messages tasks: # TEST CASE 0 # Note: Create a self-signed cert just for the "unit" test. @@ -388,3 +388,6 @@ - name: Check ports managed by firewall and selinux include_tasks: tasks/check_firewall_selinux.yml + + - name: Assert {{ __default_system_log }} + include_tasks: tasks/assert_varlogmessages.yml diff --git a/tests/tests_remote.yml b/tests/tests_remote.yml index 05ef6c90..709db90f 100644 --- a/tests/tests_remote.yml +++ b/tests/tests_remote.yml @@ -11,7 +11,7 @@ /etc/rsyslog.d/30-output-files-remote_files_output0.conf __test_output_remote1: >- /etc/rsyslog.d/30-output-files-remote_files_output1.conf - + __default_system_log: /var/log/messages tasks: # TEST CASE 0 - name: "TEST CASE 0; Test configuration inputs from the remote rsyslog @@ -242,3 +242,6 @@ - name: Check ports managed by firewall and selinux include_tasks: tasks/check_firewall_selinux.yml + + - name: Assert {{ __default_system_log }} + include_tasks: tasks/assert_varlogmessages.yml diff --git a/tests/tests_server.yml b/tests/tests_server.yml index a046ff85..cc90faeb 100644 --- a/tests/tests_server.yml +++ b/tests/tests_server.yml @@ -20,7 +20,7 @@ __test_server_tcp: /etc/rsyslog.d/11-input-remote-remote_tcp.conf __test_server_udp: /etc/rsyslog.d/11-input-remote-remote_udp.conf __expected_error: "Error: remote_tcp_0 and remote_tcp_1 conflict." - + __default_system_log: /var/log/messages tasks: # TEST CASE 0 # Note: Create a self-signed cert just for the "unit" test. @@ -223,3 +223,6 @@ - "{{ __test_ca_cert }}" - "{{ __test_key }}" - "{{ __test_cert }}" + + - name: Assert {{ __default_system_log }} + include_tasks: tasks/assert_varlogmessages.yml From 408a8c1952d6722aca99cb371e34a132d0aaa8eb Mon Sep 17 00:00:00 2001 From: Sergei Petrosian Date: Mon, 27 May 2024 20:36:38 +0200 Subject: [PATCH 2/7] Fix tests on RHEL 7 --- roles/rsyslog/templates/input_relp.j2 | 3 +++ tests/tests_basics_forwards.yml | 1 + tests/tests_combination.yml | 1 - tests/tests_relp.yml | 3 +++ 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/roles/rsyslog/templates/input_relp.j2 b/roles/rsyslog/templates/input_relp.j2 index 2efeb885..76f04eef 100644 --- a/roles/rsyslog/templates/input_relp.j2 +++ b/roles/rsyslog/templates/input_relp.j2 @@ -1,7 +1,10 @@ input(name="{{ __rsyslog_input.name }}" type="imrelp" port="{{ __rsyslog_input.port | d(20514) | int }}" +{% if ansible_distribution in ['CentOS', 'RedHat'] and + ansible_distribution_version is version('7', '==') %} maxDataSize="{{ __rsyslog_input.max_data_size | d(logging_max_message_size) | int }}" +{% endif %} {% if __rsyslog_input.tls | default(true) %} {% if __rsyslog_input.ca_cert is defined %} {% set __cacert = __rsyslog_input.ca_cert %} diff --git a/tests/tests_basics_forwards.yml b/tests/tests_basics_forwards.yml index 0372c573..9f5392bf 100644 --- a/tests/tests_basics_forwards.yml +++ b/tests/tests_basics_forwards.yml @@ -94,6 +94,7 @@ target: host.domain - name: forwards_no_severity_and_facility_protocol_port_target type: forwards + target: host.domain - target: no_name.localdomain type: forwards logging_inputs: diff --git a/tests/tests_combination.yml b/tests/tests_combination.yml index 6a341a6d..7c2fba5d 100644 --- a/tests/tests_combination.yml +++ b/tests/tests_combination.yml @@ -52,7 +52,6 @@ - name: "{{ __test_tag }}" type: files input_log_path: "{{ __test_inputfiles_dir }}/*.log" - endmsg_regex: xyz - name: basic_input type: basics ratelimit_burst: 33333 diff --git a/tests/tests_relp.yml b/tests/tests_relp.yml index 1d36bbfa..4524e341 100644 --- a/tests/tests_relp.yml +++ b/tests/tests_relp.yml @@ -188,6 +188,9 @@ changed_when: false register: __result failed_when: __result.stdout != "1" + when: + - ansible_distribution in ['CentOS', 'RedHat'] + - ansible_distribution_version is version('7', '=') - name: Check preserveFQDN is on command: grep -c 'preserveFQDN="on"' {{ __test_relp_global }} From 941b12d7735feb2844818bf7b143d2591d7ac348 Mon Sep 17 00:00:00 2001 From: Sergei Petrosian Date: Wed, 29 May 2024 12:32:58 +0200 Subject: [PATCH 3/7] Do not start service earlier than necessary --- roles/rsyslog/tasks/main_core.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/rsyslog/tasks/main_core.yml b/roles/rsyslog/tasks/main_core.yml index 1ad7f308..deb6ac32 100644 --- a/roles/rsyslog/tasks/main_core.yml +++ b/roles/rsyslog/tasks/main_core.yml @@ -364,7 +364,6 @@ service: name: rsyslog enabled: true - state: started when: - __rsyslog_enabled | bool - not rsyslog_in_image | default(false) | bool From 947916e6d6e2bda1ee121b8cbca10be7c67e2b3b Mon Sep 17 00:00:00 2001 From: Sergei Petrosian Date: Thu, 30 May 2024 14:13:55 +0200 Subject: [PATCH 4/7] Fix omitting on EL 7 --- roles/rsyslog/templates/input_relp.j2 | 2 +- tests/tests_combination.yml | 5 +++++ tests/tests_relp.yml | 3 ++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/roles/rsyslog/templates/input_relp.j2 b/roles/rsyslog/templates/input_relp.j2 index 76f04eef..e274285f 100644 --- a/roles/rsyslog/templates/input_relp.j2 +++ b/roles/rsyslog/templates/input_relp.j2 @@ -2,7 +2,7 @@ input(name="{{ __rsyslog_input.name }}" type="imrelp" port="{{ __rsyslog_input.port | d(20514) | int }}" {% if ansible_distribution in ['CentOS', 'RedHat'] and - ansible_distribution_version is version('7', '==') %} + ansible_distribution_major_version is version('7', '==') %} maxDataSize="{{ __rsyslog_input.max_data_size | d(logging_max_message_size) | int }}" {% endif %} {% if __rsyslog_input.tls | default(true) %} diff --git a/tests/tests_combination.yml b/tests/tests_combination.yml index 7c2fba5d..fcaac80a 100644 --- a/tests/tests_combination.yml +++ b/tests/tests_combination.yml @@ -52,6 +52,11 @@ - name: "{{ __test_tag }}" type: files input_log_path: "{{ __test_inputfiles_dir }}/*.log" + # Not supported on EL 7 + endmsg_regex: "{{ omit + if ansible_distribution in ['CentOS', 'RedHat'] and + ansible_distribution_major_version is version('7', '==') + else 'xyz' }}" - name: basic_input type: basics ratelimit_burst: 33333 diff --git a/tests/tests_relp.yml b/tests/tests_relp.yml index 4524e341..d2a07803 100644 --- a/tests/tests_relp.yml +++ b/tests/tests_relp.yml @@ -181,6 +181,7 @@ failed_when: __result.stdout != "1" # yamllint enable rule:line-length + # maxDataSize is not supported on EL 7 - name: Check maxDataSize is logging_max_message_size command: >- grep -c 'maxDataSize="{{ logging_max_message_size }}"' @@ -190,7 +191,7 @@ failed_when: __result.stdout != "1" when: - ansible_distribution in ['CentOS', 'RedHat'] - - ansible_distribution_version is version('7', '=') + - ansible_distribution_major_version is version('7', '=') - name: Check preserveFQDN is on command: grep -c 'preserveFQDN="on"' {{ __test_relp_global }} From 1a9e3c6bd0cb67d3546f282736f10583f1934dd8 Mon Sep 17 00:00:00 2001 From: Sergei Petrosian Date: Thu, 30 May 2024 15:49:08 +0200 Subject: [PATCH 5/7] Don't check errors in ovirt_elasticsearch.yml due to expected errors --- tests/tests_ovirt_elasticsearch.yml | 9 +++++++-- tests/tests_relp.yml | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/tests_ovirt_elasticsearch.yml b/tests/tests_ovirt_elasticsearch.yml index 45df747d..b55859a0 100644 --- a/tests/tests_ovirt_elasticsearch.yml +++ b/tests/tests_ovirt_elasticsearch.yml @@ -377,5 +377,10 @@ not waiting for normal sync points" meta: flush_handlers - - name: Assert {{ __default_system_log }} - include_tasks: tasks/assert_varlogmessages.yml + # This tests results in expected error in /var/log/messages on EL 7 + # Errors are that /etc/rsyslog.d/es-ca.crt and /etc/rsyslog.d/es-cert.pem + # do not exist. + - name: Remove {{ __default_system_log }} + file: + path: "{{ __default_system_log }}" + state: absent diff --git a/tests/tests_relp.yml b/tests/tests_relp.yml index d2a07803..412c4737 100644 --- a/tests/tests_relp.yml +++ b/tests/tests_relp.yml @@ -191,7 +191,7 @@ failed_when: __result.stdout != "1" when: - ansible_distribution in ['CentOS', 'RedHat'] - - ansible_distribution_major_version is version('7', '=') + - ansible_distribution_major_version is version('7', '>') - name: Check preserveFQDN is on command: grep -c 'preserveFQDN="on"' {{ __test_relp_global }} From 4ed0583343022cc3c92c51f0cbe176c2f6b2cfe5 Mon Sep 17 00:00:00 2001 From: Sergei Petrosian Date: Tue, 4 Jun 2024 09:04:04 +0200 Subject: [PATCH 6/7] Fix EL 7 maxdatasize incorrect condition --- roles/rsyslog/templates/input_relp.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/rsyslog/templates/input_relp.j2 b/roles/rsyslog/templates/input_relp.j2 index e274285f..7196e8cb 100644 --- a/roles/rsyslog/templates/input_relp.j2 +++ b/roles/rsyslog/templates/input_relp.j2 @@ -2,7 +2,7 @@ input(name="{{ __rsyslog_input.name }}" type="imrelp" port="{{ __rsyslog_input.port | d(20514) | int }}" {% if ansible_distribution in ['CentOS', 'RedHat'] and - ansible_distribution_major_version is version('7', '==') %} + ansible_distribution_major_version is version('7', '>') %} maxDataSize="{{ __rsyslog_input.max_data_size | d(logging_max_message_size) | int }}" {% endif %} {% if __rsyslog_input.tls | default(true) %} From a789904b3038843e2ae0a45a70184aa089194d8f Mon Sep 17 00:00:00 2001 From: Sergei Petrosian Date: Tue, 4 Jun 2024 10:58:13 +0200 Subject: [PATCH 7/7] Print possible errors in {{ __default_system_log }} --- tests/tasks/assert_varlogmessages.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/tasks/assert_varlogmessages.yml b/tests/tasks/assert_varlogmessages.yml index 7cc26eb9..b7dcd6ea 100644 --- a/tests/tasks/assert_varlogmessages.yml +++ b/tests/tasks/assert_varlogmessages.yml @@ -1,11 +1,20 @@ # SPDX-License-Identifier: MIT --- - name: Get content of {{ __default_system_log }} - command: cat {{ __default_system_log }} + shell: cat {{ __default_system_log }} || true register: __default_system_log_content changed_when: false no_log: true +- name: Print possible errors in {{ __default_system_log }} + vars: + errors: >- + {{ __default_system_log_content.stdout + | regex_search('rsyslogd: error.*') }} + debug: + var: errors + when: errors | length > 0 + # /var/log/messages sometimes contains errors not visible in journalctl - name: Ensure no errors in {{ __default_system_log }} assert: